2024-11-19 08:39:46,964 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 08:39:46,984 main DEBUG Took 0.015654 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 08:39:46,985 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 08:39:46,985 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 08:39:46,986 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 08:39:46,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:46,997 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 08:39:47,016 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,019 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,020 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,022 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,023 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,025 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,026 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,027 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,028 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,029 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,030 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,031 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,031 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 08:39:47,033 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,033 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 08:39:47,035 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 08:39:47,037 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 08:39:47,039 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 08:39:47,040 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 08:39:47,042 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 08:39:47,043 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 08:39:47,052 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 08:39:47,055 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 08:39:47,057 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 08:39:47,057 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 08:39:47,058 main DEBUG createAppenders(={Console}) 2024-11-19 08:39:47,059 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 08:39:47,059 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 08:39:47,059 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 08:39:47,060 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 08:39:47,060 main DEBUG OutputStream closed 2024-11-19 08:39:47,060 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 08:39:47,061 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 08:39:47,061 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 08:39:47,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 08:39:47,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 08:39:47,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 08:39:47,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 08:39:47,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 08:39:47,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 08:39:47,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 08:39:47,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 08:39:47,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 08:39:47,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 08:39:47,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 08:39:47,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 08:39:47,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 08:39:47,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 08:39:47,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 08:39:47,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 08:39:47,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 08:39:47,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 08:39:47,192 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 08:39:47,192 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 08:39:47,193 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 08:39:47,194 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T08:39:47,613 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea 2024-11-19 08:39:47,619 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 08:39:47,621 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T08:39:47,637 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-19T08:39:47,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T08:39:47,687 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07, deleteOnExit=true 2024-11-19T08:39:47,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T08:39:47,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/test.cache.data in system properties and HBase conf 2024-11-19T08:39:47,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T08:39:47,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/hadoop.log.dir in system properties and HBase conf 2024-11-19T08:39:47,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T08:39:47,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T08:39:47,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T08:39:47,799 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T08:39:47,923 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T08:39:47,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T08:39:47,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T08:39:47,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T08:39:47,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T08:39:47,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T08:39:47,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T08:39:47,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T08:39:47,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T08:39:47,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T08:39:47,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/nfs.dump.dir in system properties and HBase conf 2024-11-19T08:39:47,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/java.io.tmpdir in system properties and HBase conf 2024-11-19T08:39:47,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T08:39:47,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T08:39:47,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T08:39:49,216 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T08:39:49,301 INFO [Time-limited test {}] log.Log(170): Logging initialized @3281ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T08:39:49,404 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T08:39:49,504 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T08:39:49,553 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T08:39:49,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T08:39:49,556 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T08:39:49,577 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T08:39:49,581 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/hadoop.log.dir/,AVAILABLE} 2024-11-19T08:39:49,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T08:39:49,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/java.io.tmpdir/jetty-localhost-37915-hadoop-hdfs-3_4_1-tests_jar-_-any-4157388919532995021/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T08:39:49,822 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37915} 2024-11-19T08:39:49,822 INFO [Time-limited test {}] server.Server(415): Started @3803ms 2024-11-19T08:39:50,420 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T08:39:50,428 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T08:39:50,429 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T08:39:50,429 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T08:39:50,429 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T08:39:50,430 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/hadoop.log.dir/,AVAILABLE} 2024-11-19T08:39:50,430 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T08:39:50,527 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/java.io.tmpdir/jetty-localhost-40885-hadoop-hdfs-3_4_1-tests_jar-_-any-16366203878439235499/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T08:39:50,528 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40885} 2024-11-19T08:39:50,528 INFO [Time-limited test {}] server.Server(415): Started @4509ms 2024-11-19T08:39:50,575 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T08:39:51,440 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/data/data2/current/BP-951815767-172.17.0.2-1732005588737/current, will proceed with Du for space computation calculation, 2024-11-19T08:39:51,440 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/data/data1/current/BP-951815767-172.17.0.2-1732005588737/current, will proceed with Du for space computation calculation, 2024-11-19T08:39:51,472 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T08:39:51,515 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9890080451f8f0d with lease ID 0xcee9d0a39958d94d: Processing first storage report for DS-d43425d3-b09b-4456-89b9-3f0608700b2a from datanode DatanodeRegistration(127.0.0.1:36115, datanodeUuid=41be1e1c-e9a1-44c9-893b-730bc6b67745, infoPort=44177, infoSecurePort=0, ipcPort=45969, storageInfo=lv=-57;cid=testClusterID;nsid=1755016699;c=1732005588737) 2024-11-19T08:39:51,519 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9890080451f8f0d with lease ID 0xcee9d0a39958d94d: from storage DS-d43425d3-b09b-4456-89b9-3f0608700b2a node DatanodeRegistration(127.0.0.1:36115, datanodeUuid=41be1e1c-e9a1-44c9-893b-730bc6b67745, infoPort=44177, infoSecurePort=0, ipcPort=45969, storageInfo=lv=-57;cid=testClusterID;nsid=1755016699;c=1732005588737), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-11-19T08:39:51,520 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9890080451f8f0d with lease ID 0xcee9d0a39958d94d: Processing first storage report for DS-04352b20-0dd7-427b-b109-5dbc53fee13e from datanode DatanodeRegistration(127.0.0.1:36115, datanodeUuid=41be1e1c-e9a1-44c9-893b-730bc6b67745, infoPort=44177, infoSecurePort=0, ipcPort=45969, storageInfo=lv=-57;cid=testClusterID;nsid=1755016699;c=1732005588737) 2024-11-19T08:39:51,520 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9890080451f8f0d with lease ID 0xcee9d0a39958d94d: from storage DS-04352b20-0dd7-427b-b109-5dbc53fee13e node DatanodeRegistration(127.0.0.1:36115, datanodeUuid=41be1e1c-e9a1-44c9-893b-730bc6b67745, infoPort=44177, infoSecurePort=0, ipcPort=45969, storageInfo=lv=-57;cid=testClusterID;nsid=1755016699;c=1732005588737), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T08:39:51,527 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea 2024-11-19T08:39:51,592 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/zookeeper_0, clientPort=55285, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T08:39:51,601 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55285 2024-11-19T08:39:51,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T08:39:51,616 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T08:39:51,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741825_1001 (size=7) 2024-11-19T08:39:52,241 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a with version=8 2024-11-19T08:39:52,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/hbase-staging 2024-11-19T08:39:52,324 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T08:39:52,596 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3ab37fa97a98:0 server-side Connection retries=45 2024-11-19T08:39:52,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T08:39:52,609 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T08:39:52,614 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T08:39:52,614 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T08:39:52,614 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T08:39:52,763 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T08:39:52,832 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T08:39:52,845 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T08:39:52,851 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T08:39:52,888 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1536 (auto-detected) 2024-11-19T08:39:52,890 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T08:39:52,916 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36325 2024-11-19T08:39:52,945 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36325 connecting to ZooKeeper ensemble=127.0.0.1:55285 2024-11-19T08:39:53,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:363250x0, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T08:39:53,048 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36325-0x10153898cdd0000 connected 2024-11-19T08:39:53,134 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T08:39:53,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T08:39:53,147 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T08:39:53,150 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a, hbase.cluster.distributed=false 2024-11-19T08:39:53,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T08:39:53,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36325 2024-11-19T08:39:53,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36325 2024-11-19T08:39:53,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36325 2024-11-19T08:39:53,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36325 2024-11-19T08:39:53,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36325 2024-11-19T08:39:53,303 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3ab37fa97a98:0 server-side Connection retries=45 2024-11-19T08:39:53,305 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T08:39:53,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T08:39:53,306 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T08:39:53,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T08:39:53,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T08:39:53,310 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T08:39:53,313 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T08:39:53,314 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34165 2024-11-19T08:39:53,317 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34165 connecting to ZooKeeper ensemble=127.0.0.1:55285 2024-11-19T08:39:53,318 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T08:39:53,321 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T08:39:53,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341650x0, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T08:39:53,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:341650x0, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T08:39:53,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34165-0x10153898cdd0001 connected 2024-11-19T08:39:53,365 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T08:39:53,371 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T08:39:53,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T08:39:53,378 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T08:39:53,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34165 2024-11-19T08:39:53,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34165 2024-11-19T08:39:53,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34165 2024-11-19T08:39:53,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34165 2024-11-19T08:39:53,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34165 2024-11-19T08:39:53,393 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3ab37fa97a98:36325 2024-11-19T08:39:53,394 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:53,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T08:39:53,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T08:39:53,412 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:53,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T08:39:53,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:53,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:53,442 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T08:39:53,443 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3ab37fa97a98,36325,1732005592414 from backup master directory 2024-11-19T08:39:53,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T08:39:53,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:53,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T08:39:53,453 WARN [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T08:39:53,453 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:53,456 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T08:39:53,458 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T08:39:53,511 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/hbase.id] with ID: 80b3abe4-4536-4a59-95a0-4a431c025320 2024-11-19T08:39:53,511 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/.tmp/hbase.id 2024-11-19T08:39:53,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741826_1002 (size=42) 2024-11-19T08:39:53,927 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/.tmp/hbase.id]:[hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/hbase.id] 2024-11-19T08:39:53,982 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T08:39:53,990 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T08:39:54,008 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T08:39:54,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:54,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741827_1003 (size=196) 2024-11-19T08:39:54,516 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:39:54,518 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T08:39:54,530 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:39:54,534 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T08:39:54,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741828_1004 (size=1189) 2024-11-19T08:39:54,983 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store 2024-11-19T08:39:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741829_1005 (size=34) 2024-11-19T08:39:55,407 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T08:39:55,410 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:39:55,412 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T08:39:55,412 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:39:55,413 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:39:55,415 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T08:39:55,415 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:39:55,415 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:39:55,416 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732005595412Disabling compacts and flushes for region at 1732005595412Disabling writes for close at 1732005595415 (+3 ms)Writing region close event to WAL at 1732005595415Closed at 1732005595415 2024-11-19T08:39:55,419 WARN [master/3ab37fa97a98:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/.initializing 2024-11-19T08:39:55,419 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/WALs/3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:55,428 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T08:39:55,444 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3ab37fa97a98%2C36325%2C1732005592414, suffix=, logDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/WALs/3ab37fa97a98,36325,1732005592414, archiveDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/oldWALs, maxLogs=10 2024-11-19T08:39:55,467 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/WALs/3ab37fa97a98,36325,1732005592414/3ab37fa97a98%2C36325%2C1732005592414.1732005595448, exclude list is [], retry=0 2024-11-19T08:39:55,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36115,DS-d43425d3-b09b-4456-89b9-3f0608700b2a,DISK] 2024-11-19T08:39:55,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T08:39:55,530 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/WALs/3ab37fa97a98,36325,1732005592414/3ab37fa97a98%2C36325%2C1732005592414.1732005595448 2024-11-19T08:39:55,531 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44177:44177)] 2024-11-19T08:39:55,532 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:39:55,532 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:39:55,535 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,536 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T08:39:55,607 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:55,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:55,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T08:39:55,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:55,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:39:55,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T08:39:55,621 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:55,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:39:55,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T08:39:55,627 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:55,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:39:55,630 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,634 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,638 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,645 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,647 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,651 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T08:39:55,656 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T08:39:55,664 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:39:55,665 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74587292, jitterRate=0.11143726110458374}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T08:39:55,672 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732005595549Initializing all the Stores at 1732005595552 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732005595553 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005595553Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005595554 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005595554Cleaning up temporary data from old regions at 1732005595647 (+93 ms)Region opened successfully at 1732005595672 (+25 ms) 2024-11-19T08:39:55,673 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T08:39:55,711 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e10692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3ab37fa97a98/172.17.0.2:0 2024-11-19T08:39:55,746 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T08:39:55,757 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T08:39:55,757 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T08:39:55,760 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T08:39:55,762 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T08:39:55,767 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T08:39:55,768 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T08:39:55,801 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T08:39:55,813 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T08:39:55,868 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T08:39:55,870 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T08:39:55,872 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T08:39:55,882 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T08:39:55,885 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T08:39:55,888 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T08:39:55,899 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T08:39:55,901 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T08:39:55,910 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T08:39:55,929 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T08:39:55,940 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T08:39:55,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T08:39:55,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T08:39:55,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:55,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:55,979 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3ab37fa97a98,36325,1732005592414, sessionid=0x10153898cdd0000, setting cluster-up flag (Was=false) 2024-11-19T08:39:56,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:56,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:56,035 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T08:39:56,038 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:56,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:56,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:56,085 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T08:39:56,088 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:56,097 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T08:39:56,160 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T08:39:56,170 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T08:39:56,176 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T08:39:56,181 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3ab37fa97a98,36325,1732005592414 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T08:39:56,187 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(746): ClusterId : 80b3abe4-4536-4a59-95a0-4a431c025320 2024-11-19T08:39:56,188 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3ab37fa97a98:0, corePoolSize=5, maxPoolSize=5 2024-11-19T08:39:56,188 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3ab37fa97a98:0, corePoolSize=5, maxPoolSize=5 2024-11-19T08:39:56,188 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3ab37fa97a98:0, corePoolSize=5, maxPoolSize=5 2024-11-19T08:39:56,188 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3ab37fa97a98:0, corePoolSize=5, maxPoolSize=5 2024-11-19T08:39:56,188 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3ab37fa97a98:0, corePoolSize=10, maxPoolSize=10 2024-11-19T08:39:56,189 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,189 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3ab37fa97a98:0, corePoolSize=2, maxPoolSize=2 2024-11-19T08:39:56,189 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,189 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T08:39:56,190 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732005626190 2024-11-19T08:39:56,192 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T08:39:56,193 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T08:39:56,194 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T08:39:56,194 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T08:39:56,196 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T08:39:56,197 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T08:39:56,197 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T08:39:56,197 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T08:39:56,199 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:56,198 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,200 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T08:39:56,201 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T08:39:56,203 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T08:39:56,203 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T08:39:56,206 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T08:39:56,206 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T08:39:56,208 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3ab37fa97a98:0:becomeActiveMaster-HFileCleaner.large.0-1732005596208,5,FailOnTimeoutGroup] 2024-11-19T08:39:56,209 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T08:39:56,209 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3ab37fa97a98:0:becomeActiveMaster-HFileCleaner.small.0-1732005596209,5,FailOnTimeoutGroup] 2024-11-19T08:39:56,209 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T08:39:56,209 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,209 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T08:39:56,211 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741831_1007 (size=1321) 2024-11-19T08:39:56,211 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,219 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T08:39:56,220 DEBUG [RS:0;3ab37fa97a98:34165 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26783b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3ab37fa97a98/172.17.0.2:0 2024-11-19T08:39:56,232 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3ab37fa97a98:34165 2024-11-19T08:39:56,234 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T08:39:56,235 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T08:39:56,235 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T08:39:56,237 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(2659): reportForDuty to master=3ab37fa97a98,36325,1732005592414 with port=34165, startcode=1732005593269 2024-11-19T08:39:56,247 DEBUG [RS:0;3ab37fa97a98:34165 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T08:39:56,275 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33875, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T08:39:56,280 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36325 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:56,283 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36325 {}] master.ServerManager(517): Registering regionserver=3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:56,296 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:39:56,296 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44959 2024-11-19T08:39:56,296 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T08:39:56,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T08:39:56,324 DEBUG [RS:0;3ab37fa97a98:34165 {}] zookeeper.ZKUtil(111): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:56,325 WARN [RS:0;3ab37fa97a98:34165 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T08:39:56,325 INFO [RS:0;3ab37fa97a98:34165 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T08:39:56,325 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/WALs/3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:56,327 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3ab37fa97a98,34165,1732005593269] 2024-11-19T08:39:56,352 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T08:39:56,368 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T08:39:56,373 INFO [RS:0;3ab37fa97a98:34165 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T08:39:56,373 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,374 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T08:39:56,379 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T08:39:56,380 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,380 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,380 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,380 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,381 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,381 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,381 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3ab37fa97a98:0, corePoolSize=2, maxPoolSize=2 2024-11-19T08:39:56,381 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,381 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,381 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,382 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,382 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,382 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3ab37fa97a98:0, corePoolSize=1, maxPoolSize=1 2024-11-19T08:39:56,382 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3ab37fa97a98:0, corePoolSize=3, maxPoolSize=3 2024-11-19T08:39:56,382 DEBUG [RS:0;3ab37fa97a98:34165 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0, corePoolSize=3, maxPoolSize=3 2024-11-19T08:39:56,383 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,384 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,384 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,384 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,384 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,384 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,34165,1732005593269-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T08:39:56,401 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T08:39:56,403 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,34165,1732005593269-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,403 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,403 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.Replication(171): 3ab37fa97a98,34165,1732005593269 started 2024-11-19T08:39:56,421 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:56,422 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1482): Serving as 3ab37fa97a98,34165,1732005593269, RpcServer on 3ab37fa97a98/172.17.0.2:34165, sessionid=0x10153898cdd0001 2024-11-19T08:39:56,422 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T08:39:56,423 DEBUG [RS:0;3ab37fa97a98:34165 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:56,423 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3ab37fa97a98,34165,1732005593269' 2024-11-19T08:39:56,423 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T08:39:56,424 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T08:39:56,424 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T08:39:56,424 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T08:39:56,425 DEBUG [RS:0;3ab37fa97a98:34165 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:56,425 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3ab37fa97a98,34165,1732005593269' 2024-11-19T08:39:56,425 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T08:39:56,425 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T08:39:56,426 DEBUG [RS:0;3ab37fa97a98:34165 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T08:39:56,426 INFO [RS:0;3ab37fa97a98:34165 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T08:39:56,426 INFO [RS:0;3ab37fa97a98:34165 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T08:39:56,537 INFO [RS:0;3ab37fa97a98:34165 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T08:39:56,542 INFO [RS:0;3ab37fa97a98:34165 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3ab37fa97a98%2C34165%2C1732005593269, suffix=, logDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/WALs/3ab37fa97a98,34165,1732005593269, archiveDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/oldWALs, maxLogs=32 2024-11-19T08:39:56,557 DEBUG [RS:0;3ab37fa97a98:34165 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/WALs/3ab37fa97a98,34165,1732005593269/3ab37fa97a98%2C34165%2C1732005593269.1732005596544, exclude list is [], retry=0 2024-11-19T08:39:56,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36115,DS-d43425d3-b09b-4456-89b9-3f0608700b2a,DISK] 2024-11-19T08:39:56,565 INFO [RS:0;3ab37fa97a98:34165 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/WALs/3ab37fa97a98,34165,1732005593269/3ab37fa97a98%2C34165%2C1732005593269.1732005596544 2024-11-19T08:39:56,565 DEBUG [RS:0;3ab37fa97a98:34165 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44177:44177)] 2024-11-19T08:39:56,613 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T08:39:56,614 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:39:56,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741833_1009 (size=32) 2024-11-19T08:39:57,031 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:39:57,035 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T08:39:57,039 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T08:39:57,039 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,040 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,040 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T08:39:57,043 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T08:39:57,043 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T08:39:57,046 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T08:39:57,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T08:39:57,050 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T08:39:57,050 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,051 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T08:39:57,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740 2024-11-19T08:39:57,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740 2024-11-19T08:39:57,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T08:39:57,056 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T08:39:57,057 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:39:57,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T08:39:57,062 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:39:57,063 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73205158, jitterRate=0.0908418595790863}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:39:57,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732005597031Initializing all the Stores at 1732005597034 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732005597034Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732005597035 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005597035Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732005597035Cleaning up temporary data from old regions at 1732005597056 (+21 ms)Region opened successfully at 1732005597066 (+10 ms) 2024-11-19T08:39:57,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T08:39:57,066 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T08:39:57,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T08:39:57,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T08:39:57,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T08:39:57,068 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T08:39:57,068 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732005597066Disabling compacts and flushes for region at 1732005597066Disabling writes for close at 1732005597067 (+1 ms)Writing region close event to WAL at 1732005597068 (+1 ms)Closed at 1732005597068 2024-11-19T08:39:57,071 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T08:39:57,071 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T08:39:57,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T08:39:57,083 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T08:39:57,086 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T08:39:57,242 DEBUG [3ab37fa97a98:36325 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T08:39:57,255 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:57,261 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3ab37fa97a98,34165,1732005593269, state=OPENING 2024-11-19T08:39:57,284 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T08:39:57,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:57,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:39:57,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T08:39:57,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T08:39:57,297 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T08:39:57,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:39:57,485 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T08:39:57,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T08:39:57,498 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T08:39:57,498 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T08:39:57,499 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T08:39:57,502 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3ab37fa97a98%2C34165%2C1732005593269.meta, suffix=.meta, logDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/WALs/3ab37fa97a98,34165,1732005593269, archiveDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/oldWALs, maxLogs=32 2024-11-19T08:39:57,518 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/WALs/3ab37fa97a98,34165,1732005593269/3ab37fa97a98%2C34165%2C1732005593269.meta.1732005597505.meta, exclude list is [], retry=0 2024-11-19T08:39:57,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36115,DS-d43425d3-b09b-4456-89b9-3f0608700b2a,DISK] 2024-11-19T08:39:57,525 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/WALs/3ab37fa97a98,34165,1732005593269/3ab37fa97a98%2C34165%2C1732005593269.meta.1732005597505.meta 2024-11-19T08:39:57,525 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44177:44177)] 2024-11-19T08:39:57,526 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:39:57,528 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T08:39:57,531 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T08:39:57,537 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T08:39:57,541 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T08:39:57,542 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:39:57,542 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T08:39:57,542 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T08:39:57,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T08:39:57,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T08:39:57,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T08:39:57,549 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T08:39:57,549 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T08:39:57,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T08:39:57,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T08:39:57,554 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T08:39:57,554 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:57,555 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T08:39:57,556 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T08:39:57,557 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740 2024-11-19T08:39:57,560 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740 2024-11-19T08:39:57,563 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T08:39:57,563 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T08:39:57,564 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:39:57,567 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T08:39:57,569 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62967977, jitterRate=-0.06170402467250824}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:39:57,569 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T08:39:57,571 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732005597543Writing region info on filesystem at 1732005597543Initializing all the Stores at 1732005597545 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732005597545Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732005597545Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005597545Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732005597545Cleaning up temporary data from old regions at 1732005597563 (+18 ms)Running coprocessor post-open hooks at 1732005597569 (+6 ms)Region opened successfully at 1732005597571 (+2 ms) 2024-11-19T08:39:57,578 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732005597475 2024-11-19T08:39:57,590 DEBUG [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T08:39:57,591 INFO [RS_OPEN_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T08:39:57,592 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:57,595 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3ab37fa97a98,34165,1732005593269, state=OPEN 2024-11-19T08:39:57,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T08:39:57,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T08:39:57,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T08:39:57,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T08:39:57,690 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:57,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T08:39:57,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3ab37fa97a98,34165,1732005593269 in 391 msec 2024-11-19T08:39:57,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T08:39:57,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 625 msec 2024-11-19T08:39:57,708 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T08:39:57,708 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T08:39:57,726 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:39:57,727 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:39:57,744 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:39:57,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48013, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:39:57,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6400 sec 2024-11-19T08:39:57,766 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732005597766, completionTime=-1 2024-11-19T08:39:57,768 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T08:39:57,769 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T08:39:57,790 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-19T08:39:57,791 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732005657790 2024-11-19T08:39:57,791 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732005717791 2024-11-19T08:39:57,791 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-19T08:39:57,793 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,36325,1732005592414-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:57,793 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,36325,1732005592414-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:57,793 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,36325,1732005592414-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:57,794 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3ab37fa97a98:36325, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:57,795 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:57,795 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:57,800 DEBUG [master/3ab37fa97a98:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T08:39:57,821 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.368sec 2024-11-19T08:39:57,823 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T08:39:57,824 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T08:39:57,825 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T08:39:57,825 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T08:39:57,826 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T08:39:57,826 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,36325,1732005592414-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T08:39:57,827 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,36325,1732005592414-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T08:39:57,834 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T08:39:57,835 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T08:39:57,835 INFO [master/3ab37fa97a98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3ab37fa97a98,36325,1732005592414-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T08:39:57,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:39:57,904 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T08:39:57,904 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T08:39:57,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:39:57,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:39:57,917 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:39:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:39:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:39:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d86f895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:39:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:39:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:39:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:39:57,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:39:57,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:39:57,932 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:39:57,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:39:57,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:39:57,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:39:57,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:57,999 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=299, ProcessCount=11, AvailableMemoryMB=6513 2024-11-19T08:39:58,013 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T08:39:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ab37fa97a98,36325,1732005592414 2024-11-19T08:39:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65d48f8e 2024-11-19T08:39:58,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T08:39:58,026 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T08:39:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:39:58,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:39:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T08:39:58,048 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T08:39:58,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T08:39:58,052 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:58,055 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T08:39:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T08:39:58,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741835_1011 (size=959) 2024-11-19T08:39:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T08:39:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T08:39:58,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:39:58,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741836_1012 (size=53) 2024-11-19T08:39:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T08:39:58,888 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:39:58,889 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ca9bd3485db2e8c855878994330f02da, disabling compactions & flushes 2024-11-19T08:39:58,889 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:39:58,889 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:39:58,889 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. after waiting 0 ms 2024-11-19T08:39:58,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:39:58,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:39:58,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ca9bd3485db2e8c855878994330f02da: Waiting for close lock at 1732005598888Disabling compacts and flushes for region at 1732005598889 (+1 ms)Disabling writes for close at 1732005598889Writing region close event to WAL at 1732005598890 (+1 ms)Closed at 1732005598890 2024-11-19T08:39:58,895 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T08:39:58,902 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732005598895"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732005598895"}]},"ts":"1732005598895"} 2024-11-19T08:39:58,906 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T08:39:58,908 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T08:39:58,911 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005598908"}]},"ts":"1732005598908"} 2024-11-19T08:39:58,915 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T08:39:58,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, ASSIGN}] 2024-11-19T08:39:58,919 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, ASSIGN 2024-11-19T08:39:58,921 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, ASSIGN; state=OFFLINE, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=false 2024-11-19T08:39:59,073 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ca9bd3485db2e8c855878994330f02da, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:59,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, ASSIGN because future has completed 2024-11-19T08:39:59,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:39:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T08:39:59,244 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:39:59,245 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:39:59,246 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,246 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:39:59,246 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,246 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,250 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,253 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca9bd3485db2e8c855878994330f02da columnFamilyName A 2024-11-19T08:39:59,253 DEBUG [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:59,254 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] regionserver.HStore(327): Store=ca9bd3485db2e8c855878994330f02da/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:39:59,254 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,256 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca9bd3485db2e8c855878994330f02da columnFamilyName B 2024-11-19T08:39:59,256 DEBUG [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:59,257 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] regionserver.HStore(327): Store=ca9bd3485db2e8c855878994330f02da/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:39:59,257 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,259 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca9bd3485db2e8c855878994330f02da columnFamilyName C 2024-11-19T08:39:59,259 DEBUG [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:39:59,260 INFO [StoreOpener-ca9bd3485db2e8c855878994330f02da-1 {}] regionserver.HStore(327): Store=ca9bd3485db2e8c855878994330f02da/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:39:59,260 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,262 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,262 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,264 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,264 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,265 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:39:59,268 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,271 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:39:59,272 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ca9bd3485db2e8c855878994330f02da; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70164835, jitterRate=0.045537516474723816}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:39:59,272 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca9bd3485db2e8c855878994330f02da 2024-11-19T08:39:59,273 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ca9bd3485db2e8c855878994330f02da: Running coprocessor pre-open hook at 1732005599247Writing region info on filesystem at 1732005599247Initializing all the Stores at 1732005599250 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005599250Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005599250Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005599250Cleaning up temporary data from old regions at 1732005599265 (+15 ms)Running coprocessor post-open hooks at 1732005599272 (+7 ms)Region opened successfully at 1732005599273 (+1 ms) 2024-11-19T08:39:59,275 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., pid=6, masterSystemTime=1732005599235 2024-11-19T08:39:59,279 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:39:59,279 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:39:59,280 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ca9bd3485db2e8c855878994330f02da, regionState=OPEN, openSeqNum=2, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:39:59,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:39:59,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T08:39:59,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 in 206 msec 2024-11-19T08:39:59,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T08:39:59,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, ASSIGN in 373 msec 2024-11-19T08:39:59,296 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T08:39:59,297 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005599297"}]},"ts":"1732005599297"} 2024-11-19T08:39:59,300 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T08:39:59,302 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T08:39:59,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2660 sec 2024-11-19T08:40:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T08:40:00,210 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:00,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,221 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d463185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,224 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f0fa42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,226 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,233 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T08:40:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ab37fa97a98,36325,1732005592414 2024-11-19T08:40:00,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64ba8396 2024-11-19T08:40:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T08:40:00,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T08:40:00,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,244 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,251 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,254 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,258 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,270 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,273 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abd488a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,276 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103656b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,280 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbf9ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19664801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,286 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6396285d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439f6406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,291 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,292 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92f3d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:00,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:00,298 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463070d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:00,301 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5c6f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:00,306 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,306 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,306 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,308 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,311 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,312 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,312 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,314 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,315 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:00,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:00,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:00,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T08:40:00,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,330 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T08:40:00,333 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:00,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:40:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T08:40:00,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T08:40:00,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:00,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:00,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:00,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c7dad8596fa948a78cc04d3d58ff6aa6 is 50, key is test_row_0/A:col10/1732005600423/Put/seqid=0 2024-11-19T08:40:00,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005660567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005660568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005660577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005660578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005660581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741837_1013 (size=14341) 2024-11-19T08:40:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T08:40:00,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T08:40:00,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:00,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:00,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:00,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005660737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005660738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005660738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005660739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005660737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T08:40:00,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:00,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T08:40:00,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005660959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005660959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005660958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005660967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:00,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005660967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:00,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:00,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c7dad8596fa948a78cc04d3d58ff6aa6 2024-11-19T08:40:01,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T08:40:01,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:01,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/44f3966d567c4fa3b625a2688073dd85 is 50, key is test_row_0/B:col10/1732005600423/Put/seqid=0 2024-11-19T08:40:01,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741838_1014 (size=12001) 2024-11-19T08:40:01,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/44f3966d567c4fa3b625a2688073dd85 2024-11-19T08:40:01,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T08:40:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5ac9029677a5403793d6a5c73d6def4a is 50, key is test_row_0/C:col10/1732005600423/Put/seqid=0 2024-11-19T08:40:01,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741839_1015 (size=12001) 2024-11-19T08:40:01,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5ac9029677a5403793d6a5c73d6def4a 2024-11-19T08:40:01,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c7dad8596fa948a78cc04d3d58ff6aa6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7dad8596fa948a78cc04d3d58ff6aa6 2024-11-19T08:40:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005661267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7dad8596fa948a78cc04d3d58ff6aa6, entries=200, sequenceid=14, filesize=14.0 K 2024-11-19T08:40:01,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005661279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005661279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005661278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005661279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/44f3966d567c4fa3b625a2688073dd85 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/44f3966d567c4fa3b625a2688073dd85 2024-11-19T08:40:01,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/44f3966d567c4fa3b625a2688073dd85, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T08:40:01,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5ac9029677a5403793d6a5c73d6def4a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5ac9029677a5403793d6a5c73d6def4a 2024-11-19T08:40:01,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T08:40:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:01,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5ac9029677a5403793d6a5c73d6def4a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T08:40:01,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ca9bd3485db2e8c855878994330f02da in 930ms, sequenceid=14, compaction requested=false 2024-11-19T08:40:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T08:40:01,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T08:40:01,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T08:40:01,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,489 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:40:01,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/0cdaaa4e70814220a6fae79fba6e650d is 50, key is test_row_0/A:col10/1732005600525/Put/seqid=0 2024-11-19T08:40:01,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741840_1016 (size=12001) 2024-11-19T08:40:01,522 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/0cdaaa4e70814220a6fae79fba6e650d 2024-11-19T08:40:01,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/5e9c91efe0c545f6ba838f3f458138c8 is 50, key is test_row_0/B:col10/1732005600525/Put/seqid=0 2024-11-19T08:40:01,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741841_1017 (size=12001) 2024-11-19T08:40:01,579 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/5e9c91efe0c545f6ba838f3f458138c8 2024-11-19T08:40:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/3232330569014e058d5cb396f0fb90d8 is 50, key is test_row_0/C:col10/1732005600525/Put/seqid=0 2024-11-19T08:40:01,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741842_1018 (size=12001) 2024-11-19T08:40:01,640 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/3232330569014e058d5cb396f0fb90d8 2024-11-19T08:40:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/0cdaaa4e70814220a6fae79fba6e650d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/0cdaaa4e70814220a6fae79fba6e650d 2024-11-19T08:40:01,676 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/0cdaaa4e70814220a6fae79fba6e650d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T08:40:01,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/5e9c91efe0c545f6ba838f3f458138c8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/5e9c91efe0c545f6ba838f3f458138c8 2024-11-19T08:40:01,691 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/5e9c91efe0c545f6ba838f3f458138c8, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T08:40:01,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/3232330569014e058d5cb396f0fb90d8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3232330569014e058d5cb396f0fb90d8 2024-11-19T08:40:01,710 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3232330569014e058d5cb396f0fb90d8, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T08:40:01,713 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for ca9bd3485db2e8c855878994330f02da in 223ms, sequenceid=37, compaction requested=false 2024-11-19T08:40:01,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:01,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:01,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T08:40:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T08:40:01,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T08:40:01,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3810 sec 2024-11-19T08:40:01,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4040 sec 2024-11-19T08:40:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:01,812 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:01,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:40:01,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/caabbf86f3a74094b30435fd175e8a5b is 50, key is test_row_0/A:col10/1732005601801/Put/seqid=0 2024-11-19T08:40:01,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741843_1019 (size=16681) 2024-11-19T08:40:01,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/caabbf86f3a74094b30435fd175e8a5b 2024-11-19T08:40:01,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f9b942cc264640b6b67853ec13409e69 is 50, key is test_row_0/B:col10/1732005601801/Put/seqid=0 2024-11-19T08:40:01,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741844_1020 (size=12001) 2024-11-19T08:40:01,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f9b942cc264640b6b67853ec13409e69 2024-11-19T08:40:01,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7e5aec7b860d4cb19caaad1fa7ecf68c is 50, key is test_row_0/C:col10/1732005601801/Put/seqid=0 2024-11-19T08:40:01,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005661910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005661910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005661923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741845_1021 (size=12001) 2024-11-19T08:40:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005661933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005661932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:01,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7e5aec7b860d4cb19caaad1fa7ecf68c 2024-11-19T08:40:01,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/caabbf86f3a74094b30435fd175e8a5b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/caabbf86f3a74094b30435fd175e8a5b 2024-11-19T08:40:02,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/caabbf86f3a74094b30435fd175e8a5b, entries=250, sequenceid=48, filesize=16.3 K 2024-11-19T08:40:02,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f9b942cc264640b6b67853ec13409e69 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f9b942cc264640b6b67853ec13409e69 2024-11-19T08:40:02,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f9b942cc264640b6b67853ec13409e69, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T08:40:02,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7e5aec7b860d4cb19caaad1fa7ecf68c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e5aec7b860d4cb19caaad1fa7ecf68c 2024-11-19T08:40:02,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e5aec7b860d4cb19caaad1fa7ecf68c, entries=150, sequenceid=48, filesize=11.7 K 2024-11-19T08:40:02,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ca9bd3485db2e8c855878994330f02da in 240ms, sequenceid=48, compaction requested=true 2024-11-19T08:40:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:02,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:02,062 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:02,068 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:02,069 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:02,070 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:02,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:02,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:02,073 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:02,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:02,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:02,073 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:02,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T08:40:02,074 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7dad8596fa948a78cc04d3d58ff6aa6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/0cdaaa4e70814220a6fae79fba6e650d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/caabbf86f3a74094b30435fd175e8a5b] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=42.0 K 2024-11-19T08:40:02,077 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:02,078 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:02,078 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,078 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7dad8596fa948a78cc04d3d58ff6aa6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732005600400 2024-11-19T08:40:02,078 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/44f3966d567c4fa3b625a2688073dd85, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/5e9c91efe0c545f6ba838f3f458138c8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f9b942cc264640b6b67853ec13409e69] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=35.2 K 2024-11-19T08:40:02,080 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cdaaa4e70814220a6fae79fba6e650d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732005600525 2024-11-19T08:40:02,081 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f3966d567c4fa3b625a2688073dd85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732005600413 2024-11-19T08:40:02,082 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9c91efe0c545f6ba838f3f458138c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732005600525 2024-11-19T08:40:02,083 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting caabbf86f3a74094b30435fd175e8a5b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732005601801 2024-11-19T08:40:02,083 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b942cc264640b6b67853ec13409e69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732005601801 2024-11-19T08:40:02,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/405dd5c253f240a9acd1952f115078a2 is 50, key is test_row_0/A:col10/1732005601927/Put/seqid=0 2024-11-19T08:40:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005662094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005662098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005662105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005662107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005662113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,132 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:02,133 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/7e77a223f6fd497ebf89f0af03063fe7 is 50, key is test_row_0/B:col10/1732005601801/Put/seqid=0 2024-11-19T08:40:02,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741846_1022 (size=19021) 2024-11-19T08:40:02,158 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#11 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:02,159 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/8e88fdd49f2b41d39b0222124f83ab7f is 50, key is test_row_0/A:col10/1732005601801/Put/seqid=0 2024-11-19T08:40:02,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741848_1024 (size=12104) 2024-11-19T08:40:02,197 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/8e88fdd49f2b41d39b0222124f83ab7f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/8e88fdd49f2b41d39b0222124f83ab7f 2024-11-19T08:40:02,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741847_1023 (size=12104) 2024-11-19T08:40:02,223 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/7e77a223f6fd497ebf89f0af03063fe7 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7e77a223f6fd497ebf89f0af03063fe7 2024-11-19T08:40:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005662229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005662229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005662237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005662238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005662237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,254 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into 7e77a223f6fd497ebf89f0af03063fe7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:02,254 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:02,256 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into 8e88fdd49f2b41d39b0222124f83ab7f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:02,256 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:02,259 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=13, startTime=1732005602072; duration=0sec 2024-11-19T08:40:02,260 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:02,260 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:02,260 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:02,263 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:02,263 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:02,263 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,264 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5ac9029677a5403793d6a5c73d6def4a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3232330569014e058d5cb396f0fb90d8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e5aec7b860d4cb19caaad1fa7ecf68c] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=35.2 K 2024-11-19T08:40:02,265 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=13, startTime=1732005602055; duration=0sec 2024-11-19T08:40:02,265 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac9029677a5403793d6a5c73d6def4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732005600413 2024-11-19T08:40:02,267 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:02,267 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:02,282 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 3232330569014e058d5cb396f0fb90d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732005600525 2024-11-19T08:40:02,284 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5aec7b860d4cb19caaad1fa7ecf68c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732005601801 2024-11-19T08:40:02,347 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#12 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:02,347 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T08:40:02,347 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/ef637f5e92524c549e7e7f0119c2fe0f is 50, key is test_row_0/C:col10/1732005601801/Put/seqid=0 2024-11-19T08:40:02,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741849_1025 (size=12104) 2024-11-19T08:40:02,412 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/ef637f5e92524c549e7e7f0119c2fe0f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/ef637f5e92524c549e7e7f0119c2fe0f 2024-11-19T08:40:02,441 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into ef637f5e92524c549e7e7f0119c2fe0f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:02,441 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:02,441 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=13, startTime=1732005602073; duration=0sec 2024-11-19T08:40:02,442 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:02,442 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005662447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005662447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005662457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005662457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005662458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T08:40:02,487 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:02,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T08:40:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T08:40:02,493 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:02,495 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:02,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:02,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/405dd5c253f240a9acd1952f115078a2 2024-11-19T08:40:02,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/62d0003610c4422b9fffa41200a3e7ac is 50, key is test_row_0/B:col10/1732005601927/Put/seqid=0 2024-11-19T08:40:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T08:40:02,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741850_1026 (size=12001) 2024-11-19T08:40:02,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/62d0003610c4422b9fffa41200a3e7ac 2024-11-19T08:40:02,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8bbf9f8efd314fa9ba3850ae92d38997 is 50, key is test_row_0/C:col10/1732005601927/Put/seqid=0 2024-11-19T08:40:02,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:02,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741851_1027 (size=12001) 2024-11-19T08:40:02,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8bbf9f8efd314fa9ba3850ae92d38997 2024-11-19T08:40:02,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/405dd5c253f240a9acd1952f115078a2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/405dd5c253f240a9acd1952f115078a2 2024-11-19T08:40:02,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/405dd5c253f240a9acd1952f115078a2, entries=300, sequenceid=76, filesize=18.6 K 2024-11-19T08:40:02,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/62d0003610c4422b9fffa41200a3e7ac as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/62d0003610c4422b9fffa41200a3e7ac 2024-11-19T08:40:02,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/62d0003610c4422b9fffa41200a3e7ac, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T08:40:02,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8bbf9f8efd314fa9ba3850ae92d38997 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bbf9f8efd314fa9ba3850ae92d38997 2024-11-19T08:40:02,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bbf9f8efd314fa9ba3850ae92d38997, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T08:40:02,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for ca9bd3485db2e8c855878994330f02da in 675ms, sequenceid=76, compaction requested=false 2024-11-19T08:40:02,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:02,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:02,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:40:02,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/fcf9fe864c0844c2be1a226a7d8cdab0 is 50, key is test_row_0/A:col10/1732005602768/Put/seqid=0 2024-11-19T08:40:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T08:40:02,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741852_1028 (size=12001) 2024-11-19T08:40:02,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/fcf9fe864c0844c2be1a226a7d8cdab0 2024-11-19T08:40:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005662851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005662853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005662854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005662855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005662858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd is 50, key is test_row_0/B:col10/1732005602768/Put/seqid=0 2024-11-19T08:40:02,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741853_1029 (size=12001) 2024-11-19T08:40:02,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd 2024-11-19T08:40:02,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/9ad9bc6a7cba4505a10755d86d55d21d is 50, key is test_row_0/C:col10/1732005602768/Put/seqid=0 2024-11-19T08:40:02,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:02,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:02,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:02,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:02,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005662967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005662967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005662968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005662976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:02,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741854_1030 (size=12001) 2024-11-19T08:40:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005662976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,058 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T08:40:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T08:40:03,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:03,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,146 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T08:40:03,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T08:40:03,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T08:40:03,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T08:40:03,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T08:40:03,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T08:40:03,155 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T08:40:03,155 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T08:40:03,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005663179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005663187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005663187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005663197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005663207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/9ad9bc6a7cba4505a10755d86d55d21d 2024-11-19T08:40:03,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/fcf9fe864c0844c2be1a226a7d8cdab0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/fcf9fe864c0844c2be1a226a7d8cdab0 2024-11-19T08:40:03,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/fcf9fe864c0844c2be1a226a7d8cdab0, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T08:40:03,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd 2024-11-19T08:40:03,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T08:40:03,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/9ad9bc6a7cba4505a10755d86d55d21d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/9ad9bc6a7cba4505a10755d86d55d21d 2024-11-19T08:40:03,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/9ad9bc6a7cba4505a10755d86d55d21d, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T08:40:03,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ca9bd3485db2e8c855878994330f02da in 687ms, sequenceid=90, compaction requested=true 2024-11-19T08:40:03,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:03,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:03,463 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:03,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:03,463 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:03,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:03,469 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:03,469 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:03,469 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:03,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:03,469 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,470 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7e77a223f6fd497ebf89f0af03063fe7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/62d0003610c4422b9fffa41200a3e7ac, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=35.3 K 2024-11-19T08:40:03,470 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:03,470 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,470 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/8e88fdd49f2b41d39b0222124f83ab7f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/405dd5c253f240a9acd1952f115078a2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/fcf9fe864c0844c2be1a226a7d8cdab0] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=42.1 K 2024-11-19T08:40:03,472 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e88fdd49f2b41d39b0222124f83ab7f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732005601801 2024-11-19T08:40:03,473 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e77a223f6fd497ebf89f0af03063fe7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732005601801 2024-11-19T08:40:03,473 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405dd5c253f240a9acd1952f115078a2, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005601897 2024-11-19T08:40:03,474 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d0003610c4422b9fffa41200a3e7ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005601927 2024-11-19T08:40:03,475 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf9fe864c0844c2be1a226a7d8cdab0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732005602089 2024-11-19T08:40:03,477 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1f6f2c8ae24dc5b7ad6e2734b0c9bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732005602089 2024-11-19T08:40:03,510 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:03,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:40:03,514 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:03,515 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/651fce10b05b40a9bb80f173ce1be23a is 50, key is test_row_0/B:col10/1732005602768/Put/seqid=0 2024-11-19T08:40:03,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005663527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005663527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005663540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005663543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005663538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,561 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:03,563 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/de4adfb886bd48fda64f9cdf16c00f6b is 50, key is test_row_0/A:col10/1732005602768/Put/seqid=0 2024-11-19T08:40:03,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/5b049eba010c406a8fc69f199d27bb80 is 50, key is test_row_0/A:col10/1732005603507/Put/seqid=0 2024-11-19T08:40:03,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:03,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741855_1031 (size=12207) 2024-11-19T08:40:03,615 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/651fce10b05b40a9bb80f173ce1be23a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/651fce10b05b40a9bb80f173ce1be23a 2024-11-19T08:40:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T08:40:03,630 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into 651fce10b05b40a9bb80f173ce1be23a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:03,630 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:03,631 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=13, startTime=1732005603463; duration=0sec 2024-11-19T08:40:03,631 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:03,631 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:03,631 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:03,635 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:03,635 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:03,636 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,636 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/ef637f5e92524c549e7e7f0119c2fe0f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bbf9f8efd314fa9ba3850ae92d38997, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/9ad9bc6a7cba4505a10755d86d55d21d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=35.3 K 2024-11-19T08:40:03,638 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ef637f5e92524c549e7e7f0119c2fe0f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732005601801 2024-11-19T08:40:03,639 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bbf9f8efd314fa9ba3850ae92d38997, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005601927 2024-11-19T08:40:03,641 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad9bc6a7cba4505a10755d86d55d21d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732005602089 2024-11-19T08:40:03,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005663647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005663656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005663667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005663666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741856_1032 (size=12207) 2024-11-19T08:40:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005663667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,694 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/de4adfb886bd48fda64f9cdf16c00f6b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de4adfb886bd48fda64f9cdf16c00f6b 2024-11-19T08:40:03,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741857_1033 (size=12001) 2024-11-19T08:40:03,707 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:03,708 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/1c175616d1694ef1a57ef03d8461a40e is 50, key is test_row_0/C:col10/1732005602768/Put/seqid=0 2024-11-19T08:40:03,712 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into de4adfb886bd48fda64f9cdf16c00f6b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:03,712 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:03,712 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=13, startTime=1732005603463; duration=0sec 2024-11-19T08:40:03,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/5b049eba010c406a8fc69f199d27bb80 2024-11-19T08:40:03,717 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:03,717 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:03,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:03,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:03,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741858_1034 (size=12207) 2024-11-19T08:40:03,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/4584a734776b4ac9af20af3485205be9 is 50, key is test_row_0/B:col10/1732005603507/Put/seqid=0 2024-11-19T08:40:03,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741859_1035 (size=12001) 2024-11-19T08:40:03,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/4584a734776b4ac9af20af3485205be9 2024-11-19T08:40:03,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/6fab9679a2144ba189e5d17c2644fd3c is 50, key is test_row_0/C:col10/1732005603507/Put/seqid=0 2024-11-19T08:40:03,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005663856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005663867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741860_1036 (size=12001) 2024-11-19T08:40:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/6fab9679a2144ba189e5d17c2644fd3c 2024-11-19T08:40:03,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005663877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005663877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/5b049eba010c406a8fc69f199d27bb80 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5b049eba010c406a8fc69f199d27bb80 2024-11-19T08:40:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005663896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:03,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:03,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:03,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:03,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:03,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:03,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5b049eba010c406a8fc69f199d27bb80, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T08:40:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/4584a734776b4ac9af20af3485205be9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4584a734776b4ac9af20af3485205be9 2024-11-19T08:40:03,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4584a734776b4ac9af20af3485205be9, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T08:40:03,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/6fab9679a2144ba189e5d17c2644fd3c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fab9679a2144ba189e5d17c2644fd3c 2024-11-19T08:40:03,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fab9679a2144ba189e5d17c2644fd3c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T08:40:03,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for ca9bd3485db2e8c855878994330f02da in 466ms, sequenceid=117, compaction requested=false 2024-11-19T08:40:03,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:04,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T08:40:04,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:04,066 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-19T08:40:04,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/b595fc7d716c4cb6ab4d42f02bd558a6 is 50, key is test_row_0/A:col10/1732005603537/Put/seqid=0 2024-11-19T08:40:04,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741861_1037 (size=9657) 2024-11-19T08:40:04,138 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/b595fc7d716c4cb6ab4d42f02bd558a6 2024-11-19T08:40:04,174 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/1c175616d1694ef1a57ef03d8461a40e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c175616d1694ef1a57ef03d8461a40e 2024-11-19T08:40:04,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/c0d08c6d782647159b34f1d2b94b0cc2 is 50, key is test_row_0/B:col10/1732005603537/Put/seqid=0 2024-11-19T08:40:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:04,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:04,194 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into 1c175616d1694ef1a57ef03d8461a40e(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:04,194 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:04,194 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=13, startTime=1732005603469; duration=0sec 2024-11-19T08:40:04,194 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:04,194 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:04,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741862_1038 (size=9657) 2024-11-19T08:40:04,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005664280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005664280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005664283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005664298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005664294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005664397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005664396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005664406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005664406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005664416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005664606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005664616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005664617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005664617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,635 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/c0d08c6d782647159b34f1d2b94b0cc2 2024-11-19T08:40:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T08:40:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005664637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/3d46ad1ce8ed4a2dbaeffd6070ed374f is 50, key is test_row_0/C:col10/1732005603537/Put/seqid=0 2024-11-19T08:40:04,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741863_1039 (size=9657) 2024-11-19T08:40:04,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/3d46ad1ce8ed4a2dbaeffd6070ed374f 2024-11-19T08:40:04,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/b595fc7d716c4cb6ab4d42f02bd558a6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b595fc7d716c4cb6ab4d42f02bd558a6 2024-11-19T08:40:04,741 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b595fc7d716c4cb6ab4d42f02bd558a6, entries=100, sequenceid=128, filesize=9.4 K 2024-11-19T08:40:04,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/c0d08c6d782647159b34f1d2b94b0cc2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/c0d08c6d782647159b34f1d2b94b0cc2 2024-11-19T08:40:04,768 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/c0d08c6d782647159b34f1d2b94b0cc2, entries=100, sequenceid=128, filesize=9.4 K 2024-11-19T08:40:04,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/3d46ad1ce8ed4a2dbaeffd6070ed374f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3d46ad1ce8ed4a2dbaeffd6070ed374f 2024-11-19T08:40:04,785 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3d46ad1ce8ed4a2dbaeffd6070ed374f, entries=100, sequenceid=128, filesize=9.4 K 2024-11-19T08:40:04,787 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for ca9bd3485db2e8c855878994330f02da in 721ms, sequenceid=128, compaction requested=true 2024-11-19T08:40:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:04,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T08:40:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T08:40:04,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T08:40:04,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-11-19T08:40:04,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.3560 sec 2024-11-19T08:40:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:04,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:04,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-19T08:40:04,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c3bff100e5d043f380c5f98cd1e8df49 is 50, key is test_row_0/A:col10/1732005604290/Put/seqid=0 2024-11-19T08:40:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005664948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005664951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005664954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005664956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005664957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:04,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741864_1040 (size=14541) 2024-11-19T08:40:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005665067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005665068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005665068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005665068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005665276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005665277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005665276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005665277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c3bff100e5d043f380c5f98cd1e8df49 2024-11-19T08:40:05,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/a7328f263fcc40af8b890935929c62ad is 50, key is test_row_0/B:col10/1732005604290/Put/seqid=0 2024-11-19T08:40:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741865_1041 (size=12151) 2024-11-19T08:40:05,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/a7328f263fcc40af8b890935929c62ad 2024-11-19T08:40:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005665466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/6fbe11c2fb2c4cfeb7fc5c0bad698539 is 50, key is test_row_0/C:col10/1732005604290/Put/seqid=0 2024-11-19T08:40:05,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741866_1042 (size=12151) 2024-11-19T08:40:05,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/6fbe11c2fb2c4cfeb7fc5c0bad698539 2024-11-19T08:40:05,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c3bff100e5d043f380c5f98cd1e8df49 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c3bff100e5d043f380c5f98cd1e8df49 2024-11-19T08:40:05,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c3bff100e5d043f380c5f98cd1e8df49, entries=200, sequenceid=158, filesize=14.2 K 2024-11-19T08:40:05,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/a7328f263fcc40af8b890935929c62ad as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/a7328f263fcc40af8b890935929c62ad 2024-11-19T08:40:05,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/a7328f263fcc40af8b890935929c62ad, entries=150, sequenceid=158, filesize=11.9 K 2024-11-19T08:40:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/6fbe11c2fb2c4cfeb7fc5c0bad698539 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fbe11c2fb2c4cfeb7fc5c0bad698539 2024-11-19T08:40:05,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fbe11c2fb2c4cfeb7fc5c0bad698539, entries=150, sequenceid=158, filesize=11.9 K 2024-11-19T08:40:05,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for ca9bd3485db2e8c855878994330f02da in 676ms, sequenceid=158, compaction requested=true 2024-11-19T08:40:05,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:05,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:05,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:05,602 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T08:40:05,604 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:05,606 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:05,607 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:05,607 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:05,607 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de4adfb886bd48fda64f9cdf16c00f6b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5b049eba010c406a8fc69f199d27bb80, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b595fc7d716c4cb6ab4d42f02bd558a6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c3bff100e5d043f380c5f98cd1e8df49] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=47.3 K 2024-11-19T08:40:05,607 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:05,607 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:05,608 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:05,608 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/651fce10b05b40a9bb80f173ce1be23a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4584a734776b4ac9af20af3485205be9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/c0d08c6d782647159b34f1d2b94b0cc2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/a7328f263fcc40af8b890935929c62ad] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=44.9 K 2024-11-19T08:40:05,608 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting de4adfb886bd48fda64f9cdf16c00f6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732005602089 2024-11-19T08:40:05,608 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 651fce10b05b40a9bb80f173ce1be23a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732005602089 2024-11-19T08:40:05,609 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b049eba010c406a8fc69f199d27bb80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732005602845 2024-11-19T08:40:05,609 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4584a734776b4ac9af20af3485205be9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732005602845 2024-11-19T08:40:05,610 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b595fc7d716c4cb6ab4d42f02bd558a6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732005603537 2024-11-19T08:40:05,611 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d08c6d782647159b34f1d2b94b0cc2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732005603537 2024-11-19T08:40:05,611 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c3bff100e5d043f380c5f98cd1e8df49, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732005604256 2024-11-19T08:40:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:05,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:40:05,612 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7328f263fcc40af8b890935929c62ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732005604256 2024-11-19T08:40:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:05,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/3d22eb1482ff49749db70af8813ce627 is 50, key is test_row_1/A:col10/1732005605605/Put/seqid=0 2024-11-19T08:40:05,635 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:05,636 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/23e6cd9a61ec49dfadba6afcbe71a37b is 50, key is test_row_0/B:col10/1732005604290/Put/seqid=0 2024-11-19T08:40:05,650 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:05,651 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/9c1409939ac947d29a8d9f4f8017aaa5 is 50, key is test_row_0/A:col10/1732005604290/Put/seqid=0 2024-11-19T08:40:05,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741867_1043 (size=9757) 2024-11-19T08:40:05,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741868_1044 (size=12493) 2024-11-19T08:40:05,712 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/23e6cd9a61ec49dfadba6afcbe71a37b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/23e6cd9a61ec49dfadba6afcbe71a37b 2024-11-19T08:40:05,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005665708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005665719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005665721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,733 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into 23e6cd9a61ec49dfadba6afcbe71a37b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:05,733 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:05,733 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=12, startTime=1732005605602; duration=0sec 2024-11-19T08:40:05,733 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:05,733 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:05,733 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:05,736 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:05,736 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:05,737 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:05,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005665726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741869_1045 (size=12493) 2024-11-19T08:40:05,741 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c175616d1694ef1a57ef03d8461a40e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fab9679a2144ba189e5d17c2644fd3c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3d46ad1ce8ed4a2dbaeffd6070ed374f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fbe11c2fb2c4cfeb7fc5c0bad698539] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=44.9 K 2024-11-19T08:40:05,745 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c175616d1694ef1a57ef03d8461a40e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732005602089 2024-11-19T08:40:05,751 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fab9679a2144ba189e5d17c2644fd3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732005602845 2024-11-19T08:40:05,752 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d46ad1ce8ed4a2dbaeffd6070ed374f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732005603537 2024-11-19T08:40:05,753 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fbe11c2fb2c4cfeb7fc5c0bad698539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732005604256 2024-11-19T08:40:05,755 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/9c1409939ac947d29a8d9f4f8017aaa5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9c1409939ac947d29a8d9f4f8017aaa5 2024-11-19T08:40:05,772 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into 9c1409939ac947d29a8d9f4f8017aaa5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:05,772 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:05,772 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=12, startTime=1732005605602; duration=0sec 2024-11-19T08:40:05,773 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:05,773 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:05,794 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:05,795 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c570a2bda74840cfacd2a14721454c12 is 50, key is test_row_0/C:col10/1732005604290/Put/seqid=0 2024-11-19T08:40:05,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005665828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005665837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005665837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741870_1046 (size=12493) 2024-11-19T08:40:05,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005665846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:05,875 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c570a2bda74840cfacd2a14721454c12 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c570a2bda74840cfacd2a14721454c12 2024-11-19T08:40:05,889 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into c570a2bda74840cfacd2a14721454c12(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:05,889 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:05,889 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=12, startTime=1732005605603; duration=0sec 2024-11-19T08:40:05,889 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:05,889 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:06,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005666047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005666048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005666048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005666066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/3d22eb1482ff49749db70af8813ce627 2024-11-19T08:40:06,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/8909d83d5912486b8b5fc8e60b7f89c7 is 50, key is test_row_1/B:col10/1732005605605/Put/seqid=0 2024-11-19T08:40:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741871_1047 (size=9757) 2024-11-19T08:40:06,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005666356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005666357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005666366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005666376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005666486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,504 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in ca9bd3485db2e8c855878994330f02da of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/8909d83d5912486b8b5fc8e60b7f89c7 2024-11-19T08:40:06,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/2d3555ba22344463a397bc991390519f is 50, key is test_row_1/C:col10/1732005605605/Put/seqid=0 2024-11-19T08:40:06,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741872_1048 (size=9757) 2024-11-19T08:40:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T08:40:06,657 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:06,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T08:40:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T08:40:06,663 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:06,665 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:06,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T08:40:06,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T08:40:06,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:06,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:06,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:06,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:06,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005666867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005666876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005666877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005666896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:06,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T08:40:06,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:06,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:06,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:06,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T08:40:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:07,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/2d3555ba22344463a397bc991390519f 2024-11-19T08:40:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/3d22eb1482ff49749db70af8813ce627 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/3d22eb1482ff49749db70af8813ce627 2024-11-19T08:40:07,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/3d22eb1482ff49749db70af8813ce627, entries=100, sequenceid=169, filesize=9.5 K 2024-11-19T08:40:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/8909d83d5912486b8b5fc8e60b7f89c7 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8909d83d5912486b8b5fc8e60b7f89c7 2024-11-19T08:40:07,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8909d83d5912486b8b5fc8e60b7f89c7, entries=100, sequenceid=169, filesize=9.5 K 2024-11-19T08:40:07,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/2d3555ba22344463a397bc991390519f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/2d3555ba22344463a397bc991390519f 2024-11-19T08:40:07,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/2d3555ba22344463a397bc991390519f, entries=100, sequenceid=169, filesize=9.5 K 2024-11-19T08:40:07,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for ca9bd3485db2e8c855878994330f02da in 1503ms, sequenceid=169, compaction requested=false 2024-11-19T08:40:07,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:07,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T08:40:07,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:07,136 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:40:07,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/b6c78ba362574d4ea4e702806354a2b1 is 50, key is test_row_0/A:col10/1732005605708/Put/seqid=0 2024-11-19T08:40:07,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741873_1049 (size=12151) 2024-11-19T08:40:07,183 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/b6c78ba362574d4ea4e702806354a2b1 2024-11-19T08:40:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f72aa06b28aa4f5a88fc55eb5ab97d92 is 50, key is test_row_0/B:col10/1732005605708/Put/seqid=0 2024-11-19T08:40:07,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741874_1050 (size=12151) 2024-11-19T08:40:07,258 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f72aa06b28aa4f5a88fc55eb5ab97d92 2024-11-19T08:40:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/e701468631bf4825aaece86a1543b9d5 is 50, key is test_row_0/C:col10/1732005605708/Put/seqid=0 2024-11-19T08:40:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T08:40:07,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741875_1051 (size=12151) 2024-11-19T08:40:07,313 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/e701468631bf4825aaece86a1543b9d5 2024-11-19T08:40:07,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/b6c78ba362574d4ea4e702806354a2b1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b6c78ba362574d4ea4e702806354a2b1 2024-11-19T08:40:07,352 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b6c78ba362574d4ea4e702806354a2b1, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T08:40:07,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f72aa06b28aa4f5a88fc55eb5ab97d92 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f72aa06b28aa4f5a88fc55eb5ab97d92 2024-11-19T08:40:07,367 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f72aa06b28aa4f5a88fc55eb5ab97d92, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T08:40:07,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/e701468631bf4825aaece86a1543b9d5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/e701468631bf4825aaece86a1543b9d5 2024-11-19T08:40:07,384 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/e701468631bf4825aaece86a1543b9d5, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T08:40:07,385 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for ca9bd3485db2e8c855878994330f02da in 249ms, sequenceid=198, compaction requested=true 2024-11-19T08:40:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T08:40:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T08:40:07,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T08:40:07,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 724 msec 2024-11-19T08:40:07,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 741 msec 2024-11-19T08:40:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T08:40:07,796 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:07,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T08:40:07,800 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T08:40:07,803 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:07,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T08:40:07,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:07,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:40:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:07,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/7884f484fba643c692e94fc465c57ac8 is 50, key is test_row_0/A:col10/1732005607898/Put/seqid=0 2024-11-19T08:40:07,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T08:40:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:07,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:07,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:07,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741876_1052 (size=16931) 2024-11-19T08:40:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:07,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/7884f484fba643c692e94fc465c57ac8 2024-11-19T08:40:07,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ca642edb35d141899ef984cedc9df658 is 50, key is test_row_0/B:col10/1732005607898/Put/seqid=0 2024-11-19T08:40:08,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005668013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005668014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005668015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005668015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741877_1053 (size=12151) 2024-11-19T08:40:08,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ca642edb35d141899ef984cedc9df658 2024-11-19T08:40:08,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5e586ce1ce554f01a854dc16a97ba9ab is 50, key is test_row_0/C:col10/1732005607898/Put/seqid=0 2024-11-19T08:40:08,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741878_1054 (size=12151) 2024-11-19T08:40:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T08:40:08,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T08:40:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005668127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005668127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005668127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005668137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T08:40:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005668336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005668339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005668347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005668347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T08:40:08,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T08:40:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:08,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:08,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5e586ce1ce554f01a854dc16a97ba9ab 2024-11-19T08:40:08,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/7884f484fba643c692e94fc465c57ac8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/7884f484fba643c692e94fc465c57ac8 2024-11-19T08:40:08,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/7884f484fba643c692e94fc465c57ac8, entries=250, sequenceid=209, filesize=16.5 K 2024-11-19T08:40:08,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ca642edb35d141899ef984cedc9df658 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ca642edb35d141899ef984cedc9df658 2024-11-19T08:40:08,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ca642edb35d141899ef984cedc9df658, entries=150, sequenceid=209, filesize=11.9 K 2024-11-19T08:40:08,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5e586ce1ce554f01a854dc16a97ba9ab as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5e586ce1ce554f01a854dc16a97ba9ab 2024-11-19T08:40:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005668528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,532 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in ca9bd3485db2e8c855878994330f02da of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5e586ce1ce554f01a854dc16a97ba9ab, entries=150, sequenceid=209, filesize=11.9 K 2024-11-19T08:40:08,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ca9bd3485db2e8c855878994330f02da in 626ms, sequenceid=209, compaction requested=true 2024-11-19T08:40:08,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:08,536 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:08,537 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:08,537 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:08,538 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,538 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9c1409939ac947d29a8d9f4f8017aaa5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/3d22eb1482ff49749db70af8813ce627, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b6c78ba362574d4ea4e702806354a2b1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/7884f484fba643c692e94fc465c57ac8] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=50.1 K 2024-11-19T08:40:08,539 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1409939ac947d29a8d9f4f8017aaa5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732005604256 2024-11-19T08:40:08,540 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d22eb1482ff49749db70af8813ce627, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732005605605 2024-11-19T08:40:08,540 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6c78ba362574d4ea4e702806354a2b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732005605696 2024-11-19T08:40:08,540 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7884f484fba643c692e94fc465c57ac8, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732005607889 2024-11-19T08:40:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:08,551 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:08,552 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:08,553 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:08,553 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,553 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/23e6cd9a61ec49dfadba6afcbe71a37b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8909d83d5912486b8b5fc8e60b7f89c7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f72aa06b28aa4f5a88fc55eb5ab97d92, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ca642edb35d141899ef984cedc9df658] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=45.5 K 2024-11-19T08:40:08,553 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e6cd9a61ec49dfadba6afcbe71a37b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732005604256 2024-11-19T08:40:08,554 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8909d83d5912486b8b5fc8e60b7f89c7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732005605605 2024-11-19T08:40:08,554 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f72aa06b28aa4f5a88fc55eb5ab97d92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732005605696 2024-11-19T08:40:08,555 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ca642edb35d141899ef984cedc9df658, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732005607898 2024-11-19T08:40:08,560 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:08,560 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/6260b0f397684b5ebc82e7578f0d2d76 is 50, key is test_row_0/A:col10/1732005607898/Put/seqid=0 2024-11-19T08:40:08,568 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:08,569 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/4bdb9bae83e54e87bb796632ccf9f045 is 50, key is test_row_0/B:col10/1732005607898/Put/seqid=0 2024-11-19T08:40:08,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741879_1055 (size=12629) 2024-11-19T08:40:08,583 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/6260b0f397684b5ebc82e7578f0d2d76 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6260b0f397684b5ebc82e7578f0d2d76 2024-11-19T08:40:08,595 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into 6260b0f397684b5ebc82e7578f0d2d76(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:08,595 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:08,595 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=12, startTime=1732005608535; duration=0sec 2024-11-19T08:40:08,595 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:08,595 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:08,596 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:08,598 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:08,598 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:08,598 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,598 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c570a2bda74840cfacd2a14721454c12, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/2d3555ba22344463a397bc991390519f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/e701468631bf4825aaece86a1543b9d5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5e586ce1ce554f01a854dc16a97ba9ab] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=45.5 K 2024-11-19T08:40:08,600 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c570a2bda74840cfacd2a14721454c12, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732005604256 2024-11-19T08:40:08,600 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d3555ba22344463a397bc991390519f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732005605605 2024-11-19T08:40:08,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T08:40:08,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:08,601 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:40:08,603 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting e701468631bf4825aaece86a1543b9d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732005605696 2024-11-19T08:40:08,604 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e586ce1ce554f01a854dc16a97ba9ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732005607898 2024-11-19T08:40:08,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741880_1056 (size=12629) 2024-11-19T08:40:08,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/cf86cfa03c274926becaab28e9e7f128 is 50, key is test_row_0/A:col10/1732005608011/Put/seqid=0 2024-11-19T08:40:08,622 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:08,622 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5112ebe3acae497aabeffc9979397fe2 is 50, key is test_row_0/C:col10/1732005607898/Put/seqid=0 2024-11-19T08:40:08,623 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/4bdb9bae83e54e87bb796632ccf9f045 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4bdb9bae83e54e87bb796632ccf9f045 2024-11-19T08:40:08,639 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into 4bdb9bae83e54e87bb796632ccf9f045(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:08,639 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:08,640 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=12, startTime=1732005608551; duration=0sec 2024-11-19T08:40:08,640 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:08,640 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:08,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741881_1057 (size=12151) 2024-11-19T08:40:08,665 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/cf86cfa03c274926becaab28e9e7f128 2024-11-19T08:40:08,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741882_1058 (size=12629) 2024-11-19T08:40:08,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f034ac327dc64bcbb17eba8efefae9ae is 50, key is test_row_0/B:col10/1732005608011/Put/seqid=0 2024-11-19T08:40:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005668680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005668683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005668688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005668689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,711 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/5112ebe3acae497aabeffc9979397fe2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5112ebe3acae497aabeffc9979397fe2 2024-11-19T08:40:08,729 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into 5112ebe3acae497aabeffc9979397fe2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:08,729 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:08,729 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=12, startTime=1732005608551; duration=0sec 2024-11-19T08:40:08,730 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:08,730 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:08,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741883_1059 (size=12151) 2024-11-19T08:40:08,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005668796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005668796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005668806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005668806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T08:40:09,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005669007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005669007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005669016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005669021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,156 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f034ac327dc64bcbb17eba8efefae9ae 2024-11-19T08:40:09,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/28e59db38a5e4a488d7bf84f2849a5b0 is 50, key is test_row_0/C:col10/1732005608011/Put/seqid=0 2024-11-19T08:40:09,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741884_1060 (size=12151) 2024-11-19T08:40:09,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005669316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005669316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005669325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005669336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,606 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/28e59db38a5e4a488d7bf84f2849a5b0 2024-11-19T08:40:09,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/cf86cfa03c274926becaab28e9e7f128 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/cf86cfa03c274926becaab28e9e7f128 2024-11-19T08:40:09,626 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/cf86cfa03c274926becaab28e9e7f128, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T08:40:09,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f034ac327dc64bcbb17eba8efefae9ae as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f034ac327dc64bcbb17eba8efefae9ae 2024-11-19T08:40:09,637 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f034ac327dc64bcbb17eba8efefae9ae, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T08:40:09,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/28e59db38a5e4a488d7bf84f2849a5b0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/28e59db38a5e4a488d7bf84f2849a5b0 2024-11-19T08:40:09,654 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/28e59db38a5e4a488d7bf84f2849a5b0, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T08:40:09,656 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for ca9bd3485db2e8c855878994330f02da in 1055ms, sequenceid=235, compaction requested=false 2024-11-19T08:40:09,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:09,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:09,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T08:40:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T08:40:09,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T08:40:09,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-11-19T08:40:09,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.8820 sec 2024-11-19T08:40:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:09,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:09,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T08:40:09,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/929db8e5a985467fb4623be512a25720 is 50, key is test_row_0/A:col10/1732005609829/Put/seqid=0 2024-11-19T08:40:09,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741885_1061 (size=12151) 2024-11-19T08:40:09,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/929db8e5a985467fb4623be512a25720 2024-11-19T08:40:09,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ad981635795b4c2c9d57397dd77832ac is 50, key is test_row_0/B:col10/1732005609829/Put/seqid=0 2024-11-19T08:40:09,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005669879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005669886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005669887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005669888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:09,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741886_1062 (size=12151) 2024-11-19T08:40:09,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ad981635795b4c2c9d57397dd77832ac 2024-11-19T08:40:09,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c1170a5e98b34d51b3284a01717ebae5 is 50, key is test_row_0/C:col10/1732005609829/Put/seqid=0 2024-11-19T08:40:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T08:40:09,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:09,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T08:40:09,951 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T08:40:09,953 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:09,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:09,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741887_1063 (size=12151) 2024-11-19T08:40:09,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c1170a5e98b34d51b3284a01717ebae5 2024-11-19T08:40:09,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/929db8e5a985467fb4623be512a25720 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/929db8e5a985467fb4623be512a25720 2024-11-19T08:40:09,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/929db8e5a985467fb4623be512a25720, entries=150, sequenceid=252, filesize=11.9 K 2024-11-19T08:40:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ad981635795b4c2c9d57397dd77832ac as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ad981635795b4c2c9d57397dd77832ac 2024-11-19T08:40:10,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ad981635795b4c2c9d57397dd77832ac, entries=150, sequenceid=252, filesize=11.9 K 2024-11-19T08:40:10,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c1170a5e98b34d51b3284a01717ebae5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c1170a5e98b34d51b3284a01717ebae5 2024-11-19T08:40:10,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005669996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005669997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005669997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005670006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c1170a5e98b34d51b3284a01717ebae5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-19T08:40:10,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for ca9bd3485db2e8c855878994330f02da in 192ms, sequenceid=252, compaction requested=true 2024-11-19T08:40:10,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:10,023 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:10,025 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:10,025 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:10,025 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:10,025 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:10,026 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6260b0f397684b5ebc82e7578f0d2d76, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/cf86cfa03c274926becaab28e9e7f128, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/929db8e5a985467fb4623be512a25720] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=36.1 K 2024-11-19T08:40:10,026 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6260b0f397684b5ebc82e7578f0d2d76, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732005607898 2024-11-19T08:40:10,027 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:10,027 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:10,027 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:10,027 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4bdb9bae83e54e87bb796632ccf9f045, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f034ac327dc64bcbb17eba8efefae9ae, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ad981635795b4c2c9d57397dd77832ac] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=36.1 K 2024-11-19T08:40:10,028 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bdb9bae83e54e87bb796632ccf9f045, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732005607898 2024-11-19T08:40:10,028 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf86cfa03c274926becaab28e9e7f128, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732005607976 2024-11-19T08:40:10,028 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f034ac327dc64bcbb17eba8efefae9ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732005607976 2024-11-19T08:40:10,029 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ad981635795b4c2c9d57397dd77832ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005608678 2024-11-19T08:40:10,029 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929db8e5a985467fb4623be512a25720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005608678 2024-11-19T08:40:10,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:10,045 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:10,046 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/893b94e842e54ccfa36dc7a33a581b36 is 50, key is test_row_0/B:col10/1732005609829/Put/seqid=0 2024-11-19T08:40:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T08:40:10,059 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:10,060 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/f2d0b4e8b54e424bbfab131d3fa2b176 is 50, key is test_row_0/A:col10/1732005609829/Put/seqid=0 2024-11-19T08:40:10,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741889_1065 (size=12731) 2024-11-19T08:40:10,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741888_1064 (size=12731) 2024-11-19T08:40:10,106 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/893b94e842e54ccfa36dc7a33a581b36 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/893b94e842e54ccfa36dc7a33a581b36 2024-11-19T08:40:10,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T08:40:10,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:10,108 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/f2d0b4e8b54e424bbfab131d3fa2b176 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/f2d0b4e8b54e424bbfab131d3fa2b176 2024-11-19T08:40:10,108 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T08:40:10,124 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into 893b94e842e54ccfa36dc7a33a581b36(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:10,124 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:10,124 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=13, startTime=1732005610025; duration=0sec 2024-11-19T08:40:10,124 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:10,125 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:10,125 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:10,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/6863f8ee415a4abebe24e61cf349ce1a is 50, key is test_row_0/A:col10/1732005609886/Put/seqid=0 2024-11-19T08:40:10,137 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:10,137 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:10,137 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:10,137 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5112ebe3acae497aabeffc9979397fe2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/28e59db38a5e4a488d7bf84f2849a5b0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c1170a5e98b34d51b3284a01717ebae5] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=36.1 K 2024-11-19T08:40:10,138 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into f2d0b4e8b54e424bbfab131d3fa2b176(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:10,138 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:10,138 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=13, startTime=1732005610023; duration=0sec 2024-11-19T08:40:10,138 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:10,138 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:10,138 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5112ebe3acae497aabeffc9979397fe2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732005607898 2024-11-19T08:40:10,139 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 28e59db38a5e4a488d7bf84f2849a5b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732005607976 2024-11-19T08:40:10,139 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c1170a5e98b34d51b3284a01717ebae5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005608678 2024-11-19T08:40:10,164 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:10,165 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/a3bd5734b28e47f886fd68d58cd0784a is 50, key is test_row_0/C:col10/1732005609829/Put/seqid=0 2024-11-19T08:40:10,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741890_1066 (size=12301) 2024-11-19T08:40:10,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:10,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741891_1067 (size=12731) 2024-11-19T08:40:10,233 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/a3bd5734b28e47f886fd68d58cd0784a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a3bd5734b28e47f886fd68d58cd0784a 2024-11-19T08:40:10,246 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into a3bd5734b28e47f886fd68d58cd0784a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:10,246 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:10,247 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=13, startTime=1732005610041; duration=0sec 2024-11-19T08:40:10,247 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:10,247 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T08:40:10,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005670254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005670307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005670307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005670307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005670416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005670426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005670427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005670427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,573 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/6863f8ee415a4abebe24e61cf349ce1a 2024-11-19T08:40:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T08:40:10,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/b0cee236d9a648d88568b0326e7c875d is 50, key is test_row_0/B:col10/1732005609886/Put/seqid=0 2024-11-19T08:40:10,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005670627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741892_1068 (size=12301) 2024-11-19T08:40:10,637 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/b0cee236d9a648d88568b0326e7c875d 2024-11-19T08:40:10,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005670637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005670636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005670637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/16f0c5f99e6048c4a8995076712f05e2 is 50, key is test_row_0/C:col10/1732005609886/Put/seqid=0 2024-11-19T08:40:10,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741893_1069 (size=12301) 2024-11-19T08:40:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005670936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005670946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005670956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005670957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T08:40:11,105 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/16f0c5f99e6048c4a8995076712f05e2 2024-11-19T08:40:11,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/6863f8ee415a4abebe24e61cf349ce1a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6863f8ee415a4abebe24e61cf349ce1a 2024-11-19T08:40:11,121 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6863f8ee415a4abebe24e61cf349ce1a, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T08:40:11,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/b0cee236d9a648d88568b0326e7c875d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/b0cee236d9a648d88568b0326e7c875d 2024-11-19T08:40:11,132 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/b0cee236d9a648d88568b0326e7c875d, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T08:40:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/16f0c5f99e6048c4a8995076712f05e2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/16f0c5f99e6048c4a8995076712f05e2 2024-11-19T08:40:11,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/16f0c5f99e6048c4a8995076712f05e2, entries=150, sequenceid=274, filesize=12.0 K 2024-11-19T08:40:11,146 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for ca9bd3485db2e8c855878994330f02da in 1038ms, sequenceid=274, compaction requested=false 2024-11-19T08:40:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:11,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T08:40:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T08:40:11,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T08:40:11,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-19T08:40:11,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2060 sec 2024-11-19T08:40:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:11,448 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:11,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T08:40:11,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/a6240c8fd335496a8c994aff88e79e0c is 50, key is test_row_0/A:col10/1732005610241/Put/seqid=0 2024-11-19T08:40:11,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741894_1070 (size=14741) 2024-11-19T08:40:11,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/a6240c8fd335496a8c994aff88e79e0c 2024-11-19T08:40:11,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/cac3efde9a184efbb2d588de1ed0fe3e is 50, key is test_row_0/B:col10/1732005610241/Put/seqid=0 2024-11-19T08:40:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005671475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005671475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005671478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005671481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741895_1071 (size=12301) 2024-11-19T08:40:11,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005671586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005671586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005671586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005671596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005671796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005671796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005671796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005671806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:11,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/cac3efde9a184efbb2d588de1ed0fe3e 2024-11-19T08:40:11,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/062e5d15da7a47b4bfce044b1e9242b4 is 50, key is test_row_0/C:col10/1732005610241/Put/seqid=0 2024-11-19T08:40:11,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741896_1072 (size=12301) 2024-11-19T08:40:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T08:40:12,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:12,098 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T08:40:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T08:40:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005672106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005672106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005672106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,108 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,110 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:12,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005672117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T08:40:12,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T08:40:12,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:12,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:12,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:12,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:12,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/062e5d15da7a47b4bfce044b1e9242b4 2024-11-19T08:40:12,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/a6240c8fd335496a8c994aff88e79e0c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/a6240c8fd335496a8c994aff88e79e0c 2024-11-19T08:40:12,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/a6240c8fd335496a8c994aff88e79e0c, entries=200, sequenceid=294, filesize=14.4 K 2024-11-19T08:40:12,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/cac3efde9a184efbb2d588de1ed0fe3e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/cac3efde9a184efbb2d588de1ed0fe3e 2024-11-19T08:40:12,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/cac3efde9a184efbb2d588de1ed0fe3e, entries=150, sequenceid=294, filesize=12.0 K 2024-11-19T08:40:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/062e5d15da7a47b4bfce044b1e9242b4 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/062e5d15da7a47b4bfce044b1e9242b4 2024-11-19T08:40:12,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/062e5d15da7a47b4bfce044b1e9242b4, entries=150, sequenceid=294, filesize=12.0 K 2024-11-19T08:40:12,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for ca9bd3485db2e8c855878994330f02da in 922ms, sequenceid=294, compaction requested=true 2024-11-19T08:40:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:12,371 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:12,371 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:12,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:12,372 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:12,372 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:12,372 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:12,372 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:12,372 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:12,372 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:12,373 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/893b94e842e54ccfa36dc7a33a581b36, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/b0cee236d9a648d88568b0326e7c875d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/cac3efde9a184efbb2d588de1ed0fe3e] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=36.5 K 2024-11-19T08:40:12,373 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/f2d0b4e8b54e424bbfab131d3fa2b176, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6863f8ee415a4abebe24e61cf349ce1a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/a6240c8fd335496a8c994aff88e79e0c] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=38.8 K 2024-11-19T08:40:12,373 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 893b94e842e54ccfa36dc7a33a581b36, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005608678 2024-11-19T08:40:12,373 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2d0b4e8b54e424bbfab131d3fa2b176, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005608678 2024-11-19T08:40:12,374 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b0cee236d9a648d88568b0326e7c875d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732005609872 2024-11-19T08:40:12,374 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6863f8ee415a4abebe24e61cf349ce1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732005609872 2024-11-19T08:40:12,374 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting cac3efde9a184efbb2d588de1ed0fe3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732005610241 2024-11-19T08:40:12,374 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6240c8fd335496a8c994aff88e79e0c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732005610241 2024-11-19T08:40:12,393 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:12,394 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/091e77579d264313bd57dd40f3cdb88b is 50, key is test_row_0/A:col10/1732005610241/Put/seqid=0 2024-11-19T08:40:12,413 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:12,413 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/74f48243564c444cab4b62333f055bc9 is 50, key is test_row_0/B:col10/1732005610241/Put/seqid=0 2024-11-19T08:40:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T08:40:12,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T08:40:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:12,419 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T08:40:12,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741897_1073 (size=12983) 2024-11-19T08:40:12,453 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/091e77579d264313bd57dd40f3cdb88b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/091e77579d264313bd57dd40f3cdb88b 2024-11-19T08:40:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/ac4578f243ce4cf0905db7940d2c6d29 is 50, key is test_row_0/A:col10/1732005611472/Put/seqid=0 2024-11-19T08:40:12,468 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into 091e77579d264313bd57dd40f3cdb88b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:12,468 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:12,468 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=13, startTime=1732005612371; duration=0sec 2024-11-19T08:40:12,468 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:12,468 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:12,469 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:12,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741898_1074 (size=12983) 2024-11-19T08:40:12,473 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:12,474 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:12,474 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:12,474 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a3bd5734b28e47f886fd68d58cd0784a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/16f0c5f99e6048c4a8995076712f05e2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/062e5d15da7a47b4bfce044b1e9242b4] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=36.5 K 2024-11-19T08:40:12,475 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3bd5734b28e47f886fd68d58cd0784a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005608678 2024-11-19T08:40:12,476 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f0c5f99e6048c4a8995076712f05e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732005609872 2024-11-19T08:40:12,478 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062e5d15da7a47b4bfce044b1e9242b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732005610241 2024-11-19T08:40:12,500 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:12,501 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/01530c9dfa4941859a37ceb194615f6e is 50, key is test_row_0/C:col10/1732005610241/Put/seqid=0 2024-11-19T08:40:12,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741899_1075 (size=12301) 2024-11-19T08:40:12,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741900_1076 (size=12983) 2024-11-19T08:40:12,538 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/01530c9dfa4941859a37ceb194615f6e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/01530c9dfa4941859a37ceb194615f6e 2024-11-19T08:40:12,549 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into 01530c9dfa4941859a37ceb194615f6e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:12,549 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:12,549 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=13, startTime=1732005612371; duration=0sec 2024-11-19T08:40:12,549 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:12,549 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:12,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005672632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005672633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005672639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005672645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005672645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T08:40:12,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005672756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005672757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005672757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005672757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005672758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,878 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/74f48243564c444cab4b62333f055bc9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/74f48243564c444cab4b62333f055bc9 2024-11-19T08:40:12,885 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into 74f48243564c444cab4b62333f055bc9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:12,885 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:12,885 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=13, startTime=1732005612371; duration=0sec 2024-11-19T08:40:12,885 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:12,885 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:12,909 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/ac4578f243ce4cf0905db7940d2c6d29 2024-11-19T08:40:12,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/787d00a4bf1f474ab3455fed3a0ccc5b is 50, key is test_row_0/B:col10/1732005611472/Put/seqid=0 2024-11-19T08:40:12,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741901_1077 (size=12301) 2024-11-19T08:40:12,934 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/787d00a4bf1f474ab3455fed3a0ccc5b 2024-11-19T08:40:12,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/54715b8511db4209afb04d3ce8470e5d is 50, key is test_row_0/C:col10/1732005611472/Put/seqid=0 2024-11-19T08:40:12,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741902_1078 (size=12301) 2024-11-19T08:40:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005672967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005672966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005672967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005672967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005672967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T08:40:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005673276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005673276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005673276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005673277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005673279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,351 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/54715b8511db4209afb04d3ce8470e5d 2024-11-19T08:40:13,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/ac4578f243ce4cf0905db7940d2c6d29 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/ac4578f243ce4cf0905db7940d2c6d29 2024-11-19T08:40:13,383 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/ac4578f243ce4cf0905db7940d2c6d29, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T08:40:13,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/787d00a4bf1f474ab3455fed3a0ccc5b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/787d00a4bf1f474ab3455fed3a0ccc5b 2024-11-19T08:40:13,396 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/787d00a4bf1f474ab3455fed3a0ccc5b, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T08:40:13,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/54715b8511db4209afb04d3ce8470e5d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/54715b8511db4209afb04d3ce8470e5d 2024-11-19T08:40:13,409 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/54715b8511db4209afb04d3ce8470e5d, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T08:40:13,411 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for ca9bd3485db2e8c855878994330f02da in 991ms, sequenceid=313, compaction requested=false 2024-11-19T08:40:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T08:40:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T08:40:13,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T08:40:13,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3040 sec 2024-11-19T08:40:13,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3210 sec 2024-11-19T08:40:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:13,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:13,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T08:40:13,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/de6735aed920492fb1aeb5fd0b070b7f is 50, key is test_row_0/A:col10/1732005612622/Put/seqid=0 2024-11-19T08:40:13,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741903_1079 (size=14741) 2024-11-19T08:40:13,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/de6735aed920492fb1aeb5fd0b070b7f 2024-11-19T08:40:13,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005673809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005673812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005673812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005673814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005673818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f280c0d3c11d43b18841f6093483ebf7 is 50, key is test_row_0/B:col10/1732005612622/Put/seqid=0 2024-11-19T08:40:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741904_1080 (size=12301) 2024-11-19T08:40:13,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f280c0d3c11d43b18841f6093483ebf7 2024-11-19T08:40:13,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/0f4b1d3b890b406eb7d5bb5e32dcf966 is 50, key is test_row_0/C:col10/1732005612622/Put/seqid=0 2024-11-19T08:40:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741905_1081 (size=12301) 2024-11-19T08:40:13,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/0f4b1d3b890b406eb7d5bb5e32dcf966 2024-11-19T08:40:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/de6735aed920492fb1aeb5fd0b070b7f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de6735aed920492fb1aeb5fd0b070b7f 2024-11-19T08:40:13,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de6735aed920492fb1aeb5fd0b070b7f, entries=200, sequenceid=335, filesize=14.4 K 2024-11-19T08:40:13,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/f280c0d3c11d43b18841f6093483ebf7 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f280c0d3c11d43b18841f6093483ebf7 2024-11-19T08:40:13,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f280c0d3c11d43b18841f6093483ebf7, entries=150, sequenceid=335, filesize=12.0 K 2024-11-19T08:40:13,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/0f4b1d3b890b406eb7d5bb5e32dcf966 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/0f4b1d3b890b406eb7d5bb5e32dcf966 2024-11-19T08:40:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005673927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005673927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005673927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005673927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/0f4b1d3b890b406eb7d5bb5e32dcf966, entries=150, sequenceid=335, filesize=12.0 K 2024-11-19T08:40:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005673928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:13,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for ca9bd3485db2e8c855878994330f02da in 145ms, sequenceid=335, compaction requested=true 2024-11-19T08:40:13,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:13,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:13,935 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:13,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:13,935 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:13,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:13,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:13,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:13,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:13,937 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:13,937 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:13,938 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:13,938 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:13,938 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:13,938 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:13,938 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/091e77579d264313bd57dd40f3cdb88b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/ac4578f243ce4cf0905db7940d2c6d29, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de6735aed920492fb1aeb5fd0b070b7f] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=39.1 K 2024-11-19T08:40:13,938 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/74f48243564c444cab4b62333f055bc9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/787d00a4bf1f474ab3455fed3a0ccc5b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f280c0d3c11d43b18841f6093483ebf7] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=36.7 K 2024-11-19T08:40:13,939 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f48243564c444cab4b62333f055bc9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732005610241 2024-11-19T08:40:13,939 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 091e77579d264313bd57dd40f3cdb88b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732005610241 2024-11-19T08:40:13,941 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac4578f243ce4cf0905db7940d2c6d29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732005611472 2024-11-19T08:40:13,941 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 787d00a4bf1f474ab3455fed3a0ccc5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732005611472 2024-11-19T08:40:13,941 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f280c0d3c11d43b18841f6093483ebf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732005612622 2024-11-19T08:40:13,942 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6735aed920492fb1aeb5fd0b070b7f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732005612622 2024-11-19T08:40:13,964 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:13,965 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/4a140dedfc3649dea592d9088f63eb23 is 50, key is test_row_0/B:col10/1732005612622/Put/seqid=0 2024-11-19T08:40:13,973 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:13,974 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/941e6d29742c468aa47e7384aec16546 is 50, key is test_row_0/A:col10/1732005612622/Put/seqid=0 2024-11-19T08:40:13,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741906_1082 (size=13085) 2024-11-19T08:40:13,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741907_1083 (size=13085) 2024-11-19T08:40:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:14,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T08:40:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:14,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/48f6431441aa461cb035966dc90d6c9e is 50, key is test_row_0/A:col10/1732005614139/Put/seqid=0 2024-11-19T08:40:14,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741908_1084 (size=12301) 2024-11-19T08:40:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005674161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005674161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005674161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005674163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005674164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T08:40:14,246 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:14,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T08:40:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T08:40:14,249 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:14,250 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:14,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:14,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005674276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005674276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005674276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005674277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005674277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T08:40:14,392 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/4a140dedfc3649dea592d9088f63eb23 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4a140dedfc3649dea592d9088f63eb23 2024-11-19T08:40:14,399 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into 4a140dedfc3649dea592d9088f63eb23(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:14,399 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:14,399 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=13, startTime=1732005613935; duration=0sec 2024-11-19T08:40:14,400 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:14,400 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:14,400 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:14,401 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:14,401 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:14,401 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,401 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/01530c9dfa4941859a37ceb194615f6e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/54715b8511db4209afb04d3ce8470e5d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/0f4b1d3b890b406eb7d5bb5e32dcf966] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=36.7 K 2024-11-19T08:40:14,401 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 01530c9dfa4941859a37ceb194615f6e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732005610241 2024-11-19T08:40:14,402 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 54715b8511db4209afb04d3ce8470e5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732005611472 2024-11-19T08:40:14,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:14,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,404 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4b1d3b890b406eb7d5bb5e32dcf966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732005612622 2024-11-19T08:40:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,406 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/941e6d29742c468aa47e7384aec16546 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/941e6d29742c468aa47e7384aec16546 2024-11-19T08:40:14,417 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into 941e6d29742c468aa47e7384aec16546(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:14,417 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:14,417 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=13, startTime=1732005613935; duration=0sec 2024-11-19T08:40:14,417 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:14,417 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:14,435 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:14,436 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8bb9506d87154c54bb37502322f97af2 is 50, key is test_row_0/C:col10/1732005612622/Put/seqid=0 2024-11-19T08:40:14,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741909_1085 (size=13085) 2024-11-19T08:40:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005674491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005674497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005674497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005674498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005674498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/48f6431441aa461cb035966dc90d6c9e 2024-11-19T08:40:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T08:40:14,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/8085de511a4449f6b4b6986617950735 is 50, key is test_row_0/B:col10/1732005614139/Put/seqid=0 2024-11-19T08:40:14,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741910_1086 (size=12301) 2024-11-19T08:40:14,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:14,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005674806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005674806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005674807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005674807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005674807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:14,851 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8bb9506d87154c54bb37502322f97af2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bb9506d87154c54bb37502322f97af2 2024-11-19T08:40:14,860 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into 8bb9506d87154c54bb37502322f97af2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:14,860 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:14,860 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=13, startTime=1732005613936; duration=0sec 2024-11-19T08:40:14,861 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:14,861 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:14,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:14,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T08:40:14,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/8085de511a4449f6b4b6986617950735 2024-11-19T08:40:14,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/b7793e3793024479942fccb930d5ac1f is 50, key is test_row_0/C:col10/1732005614139/Put/seqid=0 2024-11-19T08:40:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741911_1087 (size=12301) 2024-11-19T08:40:15,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005675316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005675317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005675317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005675317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005675317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:15,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:15,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T08:40:15,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/b7793e3793024479942fccb930d5ac1f 2024-11-19T08:40:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/48f6431441aa461cb035966dc90d6c9e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/48f6431441aa461cb035966dc90d6c9e 2024-11-19T08:40:15,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/48f6431441aa461cb035966dc90d6c9e, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T08:40:15,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/8085de511a4449f6b4b6986617950735 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8085de511a4449f6b4b6986617950735 2024-11-19T08:40:15,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8085de511a4449f6b4b6986617950735, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T08:40:15,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/b7793e3793024479942fccb930d5ac1f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/b7793e3793024479942fccb930d5ac1f 2024-11-19T08:40:15,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/b7793e3793024479942fccb930d5ac1f, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T08:40:15,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for ca9bd3485db2e8c855878994330f02da in 1321ms, sequenceid=355, compaction requested=false 2024-11-19T08:40:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:15,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T08:40:15,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,483 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T08:40:15,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/5941fe8aeca74da985a71732ab835c41 is 50, key is test_row_0/A:col10/1732005614160/Put/seqid=0 2024-11-19T08:40:15,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741912_1088 (size=12301) 2024-11-19T08:40:15,513 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/5941fe8aeca74da985a71732ab835c41 2024-11-19T08:40:15,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ac316546576f49578d21105aafcf60a1 is 50, key is test_row_0/B:col10/1732005614160/Put/seqid=0 2024-11-19T08:40:15,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741913_1089 (size=12301) 2024-11-19T08:40:15,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ac316546576f49578d21105aafcf60a1 2024-11-19T08:40:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7c52acd65fef4aa1aa4dd574f396e0ef is 50, key is test_row_0/C:col10/1732005614160/Put/seqid=0 2024-11-19T08:40:15,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741914_1090 (size=12301) 2024-11-19T08:40:15,593 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7c52acd65fef4aa1aa4dd574f396e0ef 2024-11-19T08:40:15,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/5941fe8aeca74da985a71732ab835c41 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5941fe8aeca74da985a71732ab835c41 2024-11-19T08:40:15,611 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5941fe8aeca74da985a71732ab835c41, entries=150, sequenceid=374, filesize=12.0 K 2024-11-19T08:40:15,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ac316546576f49578d21105aafcf60a1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ac316546576f49578d21105aafcf60a1 2024-11-19T08:40:15,619 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ac316546576f49578d21105aafcf60a1, entries=150, sequenceid=374, filesize=12.0 K 2024-11-19T08:40:15,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7c52acd65fef4aa1aa4dd574f396e0ef as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7c52acd65fef4aa1aa4dd574f396e0ef 2024-11-19T08:40:15,629 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7c52acd65fef4aa1aa4dd574f396e0ef, entries=150, sequenceid=374, filesize=12.0 K 2024-11-19T08:40:15,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for ca9bd3485db2e8c855878994330f02da in 148ms, sequenceid=374, compaction requested=true 2024-11-19T08:40:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T08:40:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T08:40:15,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T08:40:15,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-11-19T08:40:15,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3910 sec 2024-11-19T08:40:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:16,349 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:16,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:40:16,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/953d5a2df5f24583b600c6309fe8f31f is 50, key is test_row_0/A:col10/1732005616345/Put/seqid=0 2024-11-19T08:40:16,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741915_1091 (size=12301) 2024-11-19T08:40:16,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/953d5a2df5f24583b600c6309fe8f31f 2024-11-19T08:40:16,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005676384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005676388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005676388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005676389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005676390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/7bf94db8507a4219abb256d2ba956be8 is 50, key is test_row_0/B:col10/1732005616345/Put/seqid=0 2024-11-19T08:40:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T08:40:16,406 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:16,406 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T08:40:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T08:40:16,409 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:16,409 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:16,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:16,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741916_1092 (size=12301) 2024-11-19T08:40:16,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/7bf94db8507a4219abb256d2ba956be8 2024-11-19T08:40:16,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/a48e78c0b8564f85a6e5eb9a14050b65 is 50, key is test_row_0/C:col10/1732005616345/Put/seqid=0 2024-11-19T08:40:16,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741917_1093 (size=12301) 2024-11-19T08:40:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005676496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005676496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005676506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005676506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005676507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T08:40:16,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005676706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005676706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005676716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005676717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005676717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:16,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T08:40:16,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/a48e78c0b8564f85a6e5eb9a14050b65 2024-11-19T08:40:16,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/953d5a2df5f24583b600c6309fe8f31f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/953d5a2df5f24583b600c6309fe8f31f 2024-11-19T08:40:16,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/953d5a2df5f24583b600c6309fe8f31f, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T08:40:16,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/7bf94db8507a4219abb256d2ba956be8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7bf94db8507a4219abb256d2ba956be8 2024-11-19T08:40:16,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:16,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7bf94db8507a4219abb256d2ba956be8, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T08:40:16,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/a48e78c0b8564f85a6e5eb9a14050b65 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a48e78c0b8564f85a6e5eb9a14050b65 2024-11-19T08:40:16,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a48e78c0b8564f85a6e5eb9a14050b65, entries=150, sequenceid=385, filesize=12.0 K 2024-11-19T08:40:16,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ca9bd3485db2e8c855878994330f02da in 533ms, sequenceid=385, compaction requested=true 2024-11-19T08:40:16,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:16,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:16,883 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:16,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:16,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:16,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:16,883 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:16,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:16,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:16,885 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:16,885 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:16,885 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/A is initiating minor compaction (all files) 2024-11-19T08:40:16,885 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/B is initiating minor compaction (all files) 2024-11-19T08:40:16,885 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/A in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,885 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/B in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,885 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/941e6d29742c468aa47e7384aec16546, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/48f6431441aa461cb035966dc90d6c9e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5941fe8aeca74da985a71732ab835c41, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/953d5a2df5f24583b600c6309fe8f31f] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=48.8 K 2024-11-19T08:40:16,885 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4a140dedfc3649dea592d9088f63eb23, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8085de511a4449f6b4b6986617950735, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ac316546576f49578d21105aafcf60a1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7bf94db8507a4219abb256d2ba956be8] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=48.8 K 2024-11-19T08:40:16,885 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941e6d29742c468aa47e7384aec16546, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732005612622 2024-11-19T08:40:16,885 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a140dedfc3649dea592d9088f63eb23, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732005612622 2024-11-19T08:40:16,886 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f6431441aa461cb035966dc90d6c9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732005613807 2024-11-19T08:40:16,886 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8085de511a4449f6b4b6986617950735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732005613807 2024-11-19T08:40:16,886 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5941fe8aeca74da985a71732ab835c41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732005614157 2024-11-19T08:40:16,886 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ac316546576f49578d21105aafcf60a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732005614157 2024-11-19T08:40:16,887 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953d5a2df5f24583b600c6309fe8f31f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732005616345 2024-11-19T08:40:16,887 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf94db8507a4219abb256d2ba956be8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732005616345 2024-11-19T08:40:16,900 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:16,901 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:16,901 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ec68013e02cb43da8876fd61e0590613 is 50, key is test_row_0/B:col10/1732005616345/Put/seqid=0 2024-11-19T08:40:16,901 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/343d985553c742ea9e648373c54a3dbf is 50, key is test_row_0/A:col10/1732005616345/Put/seqid=0 2024-11-19T08:40:16,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741919_1095 (size=13221) 2024-11-19T08:40:16,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741918_1094 (size=13221) 2024-11-19T08:40:16,919 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ec68013e02cb43da8876fd61e0590613 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ec68013e02cb43da8876fd61e0590613 2024-11-19T08:40:16,927 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/B of ca9bd3485db2e8c855878994330f02da into ec68013e02cb43da8876fd61e0590613(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:16,927 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:16,927 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/B, priority=12, startTime=1732005616883; duration=0sec 2024-11-19T08:40:16,927 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:16,927 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:16,927 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:16,930 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:16,930 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): ca9bd3485db2e8c855878994330f02da/C is initiating minor compaction (all files) 2024-11-19T08:40:16,930 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca9bd3485db2e8c855878994330f02da/C in TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:16,930 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bb9506d87154c54bb37502322f97af2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/b7793e3793024479942fccb930d5ac1f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7c52acd65fef4aa1aa4dd574f396e0ef, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a48e78c0b8564f85a6e5eb9a14050b65] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp, totalSize=48.8 K 2024-11-19T08:40:16,931 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bb9506d87154c54bb37502322f97af2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732005612622 2024-11-19T08:40:16,931 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b7793e3793024479942fccb930d5ac1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732005613807 2024-11-19T08:40:16,932 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c52acd65fef4aa1aa4dd574f396e0ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732005614157 2024-11-19T08:40:16,932 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a48e78c0b8564f85a6e5eb9a14050b65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732005616345 2024-11-19T08:40:16,958 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca9bd3485db2e8c855878994330f02da#C#compaction#83 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:16,959 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/1c57b159c6b6458996b373b83f3ef541 is 50, key is test_row_0/C:col10/1732005616345/Put/seqid=0 2024-11-19T08:40:16,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741920_1096 (size=13221) 2024-11-19T08:40:16,988 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/1c57b159c6b6458996b373b83f3ef541 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c57b159c6b6458996b373b83f3ef541 2024-11-19T08:40:16,998 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/C of ca9bd3485db2e8c855878994330f02da into 1c57b159c6b6458996b373b83f3ef541(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:16,998 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:16,998 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/C, priority=12, startTime=1732005616883; duration=0sec 2024-11-19T08:40:16,998 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:16,998 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:17,020 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:17,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:40:17,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005677029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005677030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005677031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T08:40:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005677033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005677033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c7ccfd61188745578e8d0505f178802f is 50, key is test_row_0/A:col10/1732005616374/Put/seqid=0 2024-11-19T08:40:17,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741921_1097 (size=17181) 2024-11-19T08:40:17,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c7ccfd61188745578e8d0505f178802f 2024-11-19T08:40:17,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/d42cb46fe7444cde8a6285d8f24e2005 is 50, key is test_row_0/B:col10/1732005616374/Put/seqid=0 2024-11-19T08:40:17,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741922_1098 (size=12301) 2024-11-19T08:40:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005677136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005677136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005677146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005677146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005677146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:17,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:17,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,324 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/343d985553c742ea9e648373c54a3dbf as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/343d985553c742ea9e648373c54a3dbf 2024-11-19T08:40:17,335 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca9bd3485db2e8c855878994330f02da/A of ca9bd3485db2e8c855878994330f02da into 343d985553c742ea9e648373c54a3dbf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:17,335 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:17,335 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., storeName=ca9bd3485db2e8c855878994330f02da/A, priority=12, startTime=1732005616883; duration=0sec 2024-11-19T08:40:17,335 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:17,336 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:17,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005677346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005677348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005677357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005677358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005677358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:17,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/d42cb46fe7444cde8a6285d8f24e2005 2024-11-19T08:40:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T08:40:17,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8811189c6a9b4e75ada6ca5c46e5e4c1 is 50, key is test_row_0/C:col10/1732005616374/Put/seqid=0 2024-11-19T08:40:17,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741923_1099 (size=12301) 2024-11-19T08:40:17,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8811189c6a9b4e75ada6ca5c46e5e4c1 2024-11-19T08:40:17,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/c7ccfd61188745578e8d0505f178802f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7ccfd61188745578e8d0505f178802f 2024-11-19T08:40:17,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7ccfd61188745578e8d0505f178802f, entries=250, sequenceid=414, filesize=16.8 K 2024-11-19T08:40:17,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/d42cb46fe7444cde8a6285d8f24e2005 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d42cb46fe7444cde8a6285d8f24e2005 2024-11-19T08:40:17,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d42cb46fe7444cde8a6285d8f24e2005, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T08:40:17,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8811189c6a9b4e75ada6ca5c46e5e4c1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8811189c6a9b4e75ada6ca5c46e5e4c1 2024-11-19T08:40:17,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8811189c6a9b4e75ada6ca5c46e5e4c1, entries=150, sequenceid=414, filesize=12.0 K 2024-11-19T08:40:17,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for ca9bd3485db2e8c855878994330f02da in 600ms, sequenceid=414, compaction requested=false 2024-11-19T08:40:17,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:17,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T08:40:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:17,648 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T08:40:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:17,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/999ee461afb9491082b6f0224c1ba288 is 50, key is test_row_0/A:col10/1732005617031/Put/seqid=0 2024-11-19T08:40:17,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741924_1100 (size=12301) 2024-11-19T08:40:17,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005677687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005677689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005677689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005677690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005677691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005677796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005677796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005677796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005677796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005677796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005678006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005678006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005678006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005678006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005678006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,072 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/999ee461afb9491082b6f0224c1ba288 2024-11-19T08:40:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/870e17c87916448eb1c27d08bb1bedcf is 50, key is test_row_0/B:col10/1732005617031/Put/seqid=0 2024-11-19T08:40:18,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741925_1101 (size=12301) 2024-11-19T08:40:18,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/870e17c87916448eb1c27d08bb1bedcf 2024-11-19T08:40:18,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7e21ba1439cd42fc8d1d3f2034464505 is 50, key is test_row_0/C:col10/1732005617031/Put/seqid=0 2024-11-19T08:40:18,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741926_1102 (size=12301) 2024-11-19T08:40:18,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005678316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005678316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005678316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005678316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005678316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,499 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7e21ba1439cd42fc8d1d3f2034464505 2024-11-19T08:40:18,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/999ee461afb9491082b6f0224c1ba288 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/999ee461afb9491082b6f0224c1ba288 2024-11-19T08:40:18,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/999ee461afb9491082b6f0224c1ba288, entries=150, sequenceid=425, filesize=12.0 K 2024-11-19T08:40:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/870e17c87916448eb1c27d08bb1bedcf as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/870e17c87916448eb1c27d08bb1bedcf 2024-11-19T08:40:18,517 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/870e17c87916448eb1c27d08bb1bedcf, entries=150, sequenceid=425, filesize=12.0 K 2024-11-19T08:40:18,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/7e21ba1439cd42fc8d1d3f2034464505 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e21ba1439cd42fc8d1d3f2034464505 2024-11-19T08:40:18,526 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e21ba1439cd42fc8d1d3f2034464505, entries=150, sequenceid=425, filesize=12.0 K 2024-11-19T08:40:18,527 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ca9bd3485db2e8c855878994330f02da in 879ms, sequenceid=425, compaction requested=true 2024-11-19T08:40:18,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:18,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:18,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T08:40:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T08:40:18,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T08:40:18,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-11-19T08:40:18,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1280 sec 2024-11-19T08:40:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T08:40:18,556 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:18,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T08:40:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T08:40:18,559 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:18,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:18,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T08:40:18,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T08:40:18,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:18,712 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:40:18,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/4bf15cf1d8514c77ab0d358dde35d577 is 50, key is test_row_0/A:col10/1732005617680/Put/seqid=0 2024-11-19T08:40:18,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741927_1103 (size=12301) 2024-11-19T08:40:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:18,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:18,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005678829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005678830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005678830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005678831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005678832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T08:40:18,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005678936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005678936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005678936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005678936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005678945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,131 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/4bf15cf1d8514c77ab0d358dde35d577 2024-11-19T08:40:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/d19e255f385c4391846e7369f499f2f3 is 50, key is test_row_0/B:col10/1732005617680/Put/seqid=0 2024-11-19T08:40:19,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005679146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005679146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005679146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005679147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741928_1104 (size=12301) 2024-11-19T08:40:19,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005679156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T08:40:19,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005679456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005679456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005679456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005679456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005679466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:19,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:19,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/d19e255f385c4391846e7369f499f2f3 2024-11-19T08:40:19,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8083deeafd4b4ff596fa5a7b3f3bd359 is 50, key is test_row_0/C:col10/1732005617680/Put/seqid=0 2024-11-19T08:40:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741929_1105 (size=12301) 2024-11-19T08:40:19,589 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8083deeafd4b4ff596fa5a7b3f3bd359 2024-11-19T08:40:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/4bf15cf1d8514c77ab0d358dde35d577 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/4bf15cf1d8514c77ab0d358dde35d577 2024-11-19T08:40:19,616 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/4bf15cf1d8514c77ab0d358dde35d577, entries=150, sequenceid=451, filesize=12.0 K 2024-11-19T08:40:19,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/d19e255f385c4391846e7369f499f2f3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d19e255f385c4391846e7369f499f2f3 2024-11-19T08:40:19,624 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d19e255f385c4391846e7369f499f2f3, entries=150, sequenceid=451, filesize=12.0 K 2024-11-19T08:40:19,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/8083deeafd4b4ff596fa5a7b3f3bd359 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8083deeafd4b4ff596fa5a7b3f3bd359 2024-11-19T08:40:19,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8083deeafd4b4ff596fa5a7b3f3bd359, entries=150, sequenceid=451, filesize=12.0 K 2024-11-19T08:40:19,632 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ca9bd3485db2e8c855878994330f02da in 920ms, sequenceid=451, compaction requested=true 2024-11-19T08:40:19,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:19,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:19,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T08:40:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T08:40:19,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T08:40:19,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-19T08:40:19,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.0810 sec 2024-11-19T08:40:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T08:40:19,696 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:19,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T08:40:19,698 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T08:40:19,699 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:19,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T08:40:19,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T08:40:19,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:19,853 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T08:40:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/76ddb4f26a0d434fb0c5de31eb04ad33 is 50, key is test_row_0/A:col10/1732005618828/Put/seqid=0 2024-11-19T08:40:19,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741930_1106 (size=12301) 2024-11-19T08:40:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. as already flushing 2024-11-19T08:40:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T08:40:20,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005680020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005680021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005680022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005680023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005680024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005680136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005680136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005680136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005680137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005680137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,262 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/76ddb4f26a0d434fb0c5de31eb04ad33 2024-11-19T08:40:20,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/949a4b50f92c46f1bac8cfe086ef728b is 50, key is test_row_0/B:col10/1732005618828/Put/seqid=0 2024-11-19T08:40:20,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741931_1107 (size=12301) 2024-11-19T08:40:20,308 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T08:40:20,308 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T08:40:20,308 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:20,308 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:20,310 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T08:40:20,310 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:20,313 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,313 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,313 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,314 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,314 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:20,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:20,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:20,314 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,316 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T08:40:20,316 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:20,316 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,316 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:20,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T08:40:20,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005680346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005680346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005680346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005680346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005680346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732005680656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53298 deadline: 1732005680657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53266 deadline: 1732005680656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53274 deadline: 1732005680658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53262 deadline: 1732005680657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:20,676 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/949a4b50f92c46f1bac8cfe086ef728b 2024-11-19T08:40:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c60389ea1d524d5c89f2e381d8e77464 is 50, key is test_row_0/C:col10/1732005618828/Put/seqid=0 2024-11-19T08:40:20,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741932_1108 (size=12301) 2024-11-19T08:40:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T08:40:21,093 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c60389ea1d524d5c89f2e381d8e77464 2024-11-19T08:40:21,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/76ddb4f26a0d434fb0c5de31eb04ad33 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/76ddb4f26a0d434fb0c5de31eb04ad33 2024-11-19T08:40:21,111 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/76ddb4f26a0d434fb0c5de31eb04ad33, entries=150, sequenceid=461, filesize=12.0 K 2024-11-19T08:40:21,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/949a4b50f92c46f1bac8cfe086ef728b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/949a4b50f92c46f1bac8cfe086ef728b 2024-11-19T08:40:21,120 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/949a4b50f92c46f1bac8cfe086ef728b, entries=150, sequenceid=461, filesize=12.0 K 2024-11-19T08:40:21,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/c60389ea1d524d5c89f2e381d8e77464 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c60389ea1d524d5c89f2e381d8e77464 2024-11-19T08:40:21,130 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c60389ea1d524d5c89f2e381d8e77464, entries=150, sequenceid=461, filesize=12.0 K 2024-11-19T08:40:21,131 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ca9bd3485db2e8c855878994330f02da in 1278ms, sequenceid=461, compaction requested=true 2024-11-19T08:40:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T08:40:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T08:40:21,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T08:40:21,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-11-19T08:40:21,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4380 sec 2024-11-19T08:40:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:21,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:21,168 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T08:40:21,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-19T08:40:21,168 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T08:40:21,168 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:21,168 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T08:40:21,168 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,168 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:21,168 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:21,168 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,168 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,168 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:21,168 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,168 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:21,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:21,169 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T08:40:21,170 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:21,170 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,170 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:21,171 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T08:40:21,171 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:21,171 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,171 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:21,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/763f64ede0a04bdfb84b41058ecce504 is 50, key is test_row_0/A:col10/1732005621168/Put/seqid=0 2024-11-19T08:40:21,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741933_1109 (size=12301) 2024-11-19T08:40:21,524 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T08:40:21,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/763f64ede0a04bdfb84b41058ecce504 2024-11-19T08:40:21,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ff3ab1b60b0f44f181d64622db61e1fe is 50, key is test_row_0/B:col10/1732005621168/Put/seqid=0 2024-11-19T08:40:21,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741934_1110 (size=12301) 2024-11-19T08:40:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T08:40:21,848 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:21,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3398 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3290 2024-11-19T08:40:21,849 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T08:40:21,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1533 2024-11-19T08:40:21,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4595 rows 2024-11-19T08:40:21,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1550 2024-11-19T08:40:21,850 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4640 rows 2024-11-19T08:40:21,851 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:40:21,851 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:40:21,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:21,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:21,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T08:40:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T08:40:21,869 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005621868"}]},"ts":"1732005621868"} 2024-11-19T08:40:21,870 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T08:40:21,870 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T08:40:21,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:40:21,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, UNASSIGN}] 2024-11-19T08:40:21,878 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, UNASSIGN 2024-11-19T08:40:21,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ca9bd3485db2e8c855878994330f02da, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:21,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, UNASSIGN because future has completed 2024-11-19T08:40:21,882 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:40:21,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:40:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T08:40:21,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ff3ab1b60b0f44f181d64622db61e1fe 2024-11-19T08:40:22,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/4dbf525b5053461397a1a1c2d8f05e1f is 50, key is test_row_0/C:col10/1732005621168/Put/seqid=0 2024-11-19T08:40:22,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741935_1111 (size=12301) 2024-11-19T08:40:22,040 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:22,040 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:40:22,041 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing ca9bd3485db2e8c855878994330f02da, disabling compactions & flushes 2024-11-19T08:40:22,041 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T08:40:22,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/4dbf525b5053461397a1a1c2d8f05e1f 2024-11-19T08:40:22,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/763f64ede0a04bdfb84b41058ecce504 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/763f64ede0a04bdfb84b41058ecce504 2024-11-19T08:40:22,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/763f64ede0a04bdfb84b41058ecce504, entries=150, sequenceid=491, filesize=12.0 K 2024-11-19T08:40:22,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/ff3ab1b60b0f44f181d64622db61e1fe as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ff3ab1b60b0f44f181d64622db61e1fe 2024-11-19T08:40:22,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ff3ab1b60b0f44f181d64622db61e1fe, entries=150, sequenceid=491, filesize=12.0 K 2024-11-19T08:40:22,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/4dbf525b5053461397a1a1c2d8f05e1f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4dbf525b5053461397a1a1c2d8f05e1f 2024-11-19T08:40:22,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4dbf525b5053461397a1a1c2d8f05e1f, entries=150, sequenceid=491, filesize=12.0 K 2024-11-19T08:40:22,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=6.71 KB/6870 for ca9bd3485db2e8c855878994330f02da in 1280ms, sequenceid=491, compaction requested=true 2024-11-19T08:40:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca9bd3485db2e8c855878994330f02da: 2024-11-19T08:40:22,448 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:22,448 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:22,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:22,448 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. after waiting 0 ms 2024-11-19T08:40:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:22,449 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:22,449 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. because compaction request was cancelled 2024-11-19T08:40:22,449 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. because compaction request was cancelled 2024-11-19T08:40:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:22,449 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:B 2024-11-19T08:40:22,449 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:A 2024-11-19T08:40:22,449 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. because compaction request was cancelled 2024-11-19T08:40:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca9bd3485db2e8c855878994330f02da:C, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:22,449 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing ca9bd3485db2e8c855878994330f02da 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-19T08:40:22,449 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca9bd3485db2e8c855878994330f02da:C 2024-11-19T08:40:22,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:22,453 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/9312c7c3155840818344f44f31a65450 is 50, key is test_row_0/A:col10/1732005621169/Put/seqid=0 2024-11-19T08:40:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741936_1112 (size=7415) 2024-11-19T08:40:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T08:40:22,859 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/9312c7c3155840818344f44f31a65450 2024-11-19T08:40:22,867 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/92e182cabe984cc580cd4e2338c0e11d is 50, key is test_row_0/B:col10/1732005621169/Put/seqid=0 2024-11-19T08:40:22,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741937_1113 (size=7415) 2024-11-19T08:40:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T08:40:23,272 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/92e182cabe984cc580cd4e2338c0e11d 2024-11-19T08:40:23,281 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/4acf227bdbb1461da0d8b00c0f0a9189 is 50, key is test_row_0/C:col10/1732005621169/Put/seqid=0 2024-11-19T08:40:23,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741938_1114 (size=7415) 2024-11-19T08:40:23,687 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/4acf227bdbb1461da0d8b00c0f0a9189 2024-11-19T08:40:23,704 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/A/9312c7c3155840818344f44f31a65450 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9312c7c3155840818344f44f31a65450 2024-11-19T08:40:23,712 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9312c7c3155840818344f44f31a65450, entries=50, sequenceid=495, filesize=7.2 K 2024-11-19T08:40:23,714 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/B/92e182cabe984cc580cd4e2338c0e11d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/92e182cabe984cc580cd4e2338c0e11d 2024-11-19T08:40:23,721 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/92e182cabe984cc580cd4e2338c0e11d, entries=50, sequenceid=495, filesize=7.2 K 2024-11-19T08:40:23,722 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/.tmp/C/4acf227bdbb1461da0d8b00c0f0a9189 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4acf227bdbb1461da0d8b00c0f0a9189 2024-11-19T08:40:23,729 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4acf227bdbb1461da0d8b00c0f0a9189, entries=50, sequenceid=495, filesize=7.2 K 2024-11-19T08:40:23,730 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ca9bd3485db2e8c855878994330f02da in 1281ms, sequenceid=495, compaction requested=true 2024-11-19T08:40:23,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7dad8596fa948a78cc04d3d58ff6aa6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/0cdaaa4e70814220a6fae79fba6e650d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/caabbf86f3a74094b30435fd175e8a5b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/8e88fdd49f2b41d39b0222124f83ab7f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/405dd5c253f240a9acd1952f115078a2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de4adfb886bd48fda64f9cdf16c00f6b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/fcf9fe864c0844c2be1a226a7d8cdab0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5b049eba010c406a8fc69f199d27bb80, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b595fc7d716c4cb6ab4d42f02bd558a6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c3bff100e5d043f380c5f98cd1e8df49, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9c1409939ac947d29a8d9f4f8017aaa5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/3d22eb1482ff49749db70af8813ce627, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b6c78ba362574d4ea4e702806354a2b1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/7884f484fba643c692e94fc465c57ac8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6260b0f397684b5ebc82e7578f0d2d76, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/cf86cfa03c274926becaab28e9e7f128, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/f2d0b4e8b54e424bbfab131d3fa2b176, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/929db8e5a985467fb4623be512a25720, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6863f8ee415a4abebe24e61cf349ce1a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/a6240c8fd335496a8c994aff88e79e0c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/091e77579d264313bd57dd40f3cdb88b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/ac4578f243ce4cf0905db7940d2c6d29, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de6735aed920492fb1aeb5fd0b070b7f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/941e6d29742c468aa47e7384aec16546, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/48f6431441aa461cb035966dc90d6c9e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5941fe8aeca74da985a71732ab835c41, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/953d5a2df5f24583b600c6309fe8f31f] to archive 2024-11-19T08:40:23,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:40:23,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7dad8596fa948a78cc04d3d58ff6aa6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7dad8596fa948a78cc04d3d58ff6aa6 2024-11-19T08:40:23,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/0cdaaa4e70814220a6fae79fba6e650d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/0cdaaa4e70814220a6fae79fba6e650d 2024-11-19T08:40:23,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/caabbf86f3a74094b30435fd175e8a5b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/caabbf86f3a74094b30435fd175e8a5b 2024-11-19T08:40:23,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/8e88fdd49f2b41d39b0222124f83ab7f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/8e88fdd49f2b41d39b0222124f83ab7f 2024-11-19T08:40:23,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/405dd5c253f240a9acd1952f115078a2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/405dd5c253f240a9acd1952f115078a2 2024-11-19T08:40:23,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de4adfb886bd48fda64f9cdf16c00f6b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de4adfb886bd48fda64f9cdf16c00f6b 2024-11-19T08:40:23,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/fcf9fe864c0844c2be1a226a7d8cdab0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/fcf9fe864c0844c2be1a226a7d8cdab0 2024-11-19T08:40:23,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5b049eba010c406a8fc69f199d27bb80 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5b049eba010c406a8fc69f199d27bb80 2024-11-19T08:40:23,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b595fc7d716c4cb6ab4d42f02bd558a6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b595fc7d716c4cb6ab4d42f02bd558a6 2024-11-19T08:40:23,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c3bff100e5d043f380c5f98cd1e8df49 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c3bff100e5d043f380c5f98cd1e8df49 2024-11-19T08:40:23,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9c1409939ac947d29a8d9f4f8017aaa5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9c1409939ac947d29a8d9f4f8017aaa5 2024-11-19T08:40:23,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/3d22eb1482ff49749db70af8813ce627 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/3d22eb1482ff49749db70af8813ce627 2024-11-19T08:40:23,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b6c78ba362574d4ea4e702806354a2b1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/b6c78ba362574d4ea4e702806354a2b1 2024-11-19T08:40:23,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/7884f484fba643c692e94fc465c57ac8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/7884f484fba643c692e94fc465c57ac8 2024-11-19T08:40:23,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6260b0f397684b5ebc82e7578f0d2d76 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6260b0f397684b5ebc82e7578f0d2d76 2024-11-19T08:40:23,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/cf86cfa03c274926becaab28e9e7f128 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/cf86cfa03c274926becaab28e9e7f128 2024-11-19T08:40:23,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/f2d0b4e8b54e424bbfab131d3fa2b176 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/f2d0b4e8b54e424bbfab131d3fa2b176 2024-11-19T08:40:23,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/929db8e5a985467fb4623be512a25720 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/929db8e5a985467fb4623be512a25720 2024-11-19T08:40:23,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6863f8ee415a4abebe24e61cf349ce1a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/6863f8ee415a4abebe24e61cf349ce1a 2024-11-19T08:40:23,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/a6240c8fd335496a8c994aff88e79e0c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/a6240c8fd335496a8c994aff88e79e0c 2024-11-19T08:40:23,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/091e77579d264313bd57dd40f3cdb88b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/091e77579d264313bd57dd40f3cdb88b 2024-11-19T08:40:23,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/ac4578f243ce4cf0905db7940d2c6d29 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/ac4578f243ce4cf0905db7940d2c6d29 2024-11-19T08:40:23,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de6735aed920492fb1aeb5fd0b070b7f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/de6735aed920492fb1aeb5fd0b070b7f 2024-11-19T08:40:23,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/941e6d29742c468aa47e7384aec16546 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/941e6d29742c468aa47e7384aec16546 2024-11-19T08:40:23,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/48f6431441aa461cb035966dc90d6c9e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/48f6431441aa461cb035966dc90d6c9e 2024-11-19T08:40:23,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5941fe8aeca74da985a71732ab835c41 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/5941fe8aeca74da985a71732ab835c41 2024-11-19T08:40:23,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/953d5a2df5f24583b600c6309fe8f31f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/953d5a2df5f24583b600c6309fe8f31f 2024-11-19T08:40:23,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/44f3966d567c4fa3b625a2688073dd85, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/5e9c91efe0c545f6ba838f3f458138c8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7e77a223f6fd497ebf89f0af03063fe7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f9b942cc264640b6b67853ec13409e69, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/62d0003610c4422b9fffa41200a3e7ac, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/651fce10b05b40a9bb80f173ce1be23a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4584a734776b4ac9af20af3485205be9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/c0d08c6d782647159b34f1d2b94b0cc2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/23e6cd9a61ec49dfadba6afcbe71a37b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/a7328f263fcc40af8b890935929c62ad, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8909d83d5912486b8b5fc8e60b7f89c7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f72aa06b28aa4f5a88fc55eb5ab97d92, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4bdb9bae83e54e87bb796632ccf9f045, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ca642edb35d141899ef984cedc9df658, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f034ac327dc64bcbb17eba8efefae9ae, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/893b94e842e54ccfa36dc7a33a581b36, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ad981635795b4c2c9d57397dd77832ac, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/b0cee236d9a648d88568b0326e7c875d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/74f48243564c444cab4b62333f055bc9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/cac3efde9a184efbb2d588de1ed0fe3e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/787d00a4bf1f474ab3455fed3a0ccc5b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4a140dedfc3649dea592d9088f63eb23, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f280c0d3c11d43b18841f6093483ebf7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8085de511a4449f6b4b6986617950735, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ac316546576f49578d21105aafcf60a1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7bf94db8507a4219abb256d2ba956be8] to archive 2024-11-19T08:40:23,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:40:23,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/44f3966d567c4fa3b625a2688073dd85 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/44f3966d567c4fa3b625a2688073dd85 2024-11-19T08:40:23,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/5e9c91efe0c545f6ba838f3f458138c8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/5e9c91efe0c545f6ba838f3f458138c8 2024-11-19T08:40:23,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7e77a223f6fd497ebf89f0af03063fe7 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7e77a223f6fd497ebf89f0af03063fe7 2024-11-19T08:40:23,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f9b942cc264640b6b67853ec13409e69 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f9b942cc264640b6b67853ec13409e69 2024-11-19T08:40:23,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/62d0003610c4422b9fffa41200a3e7ac to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/62d0003610c4422b9fffa41200a3e7ac 2024-11-19T08:40:23,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/651fce10b05b40a9bb80f173ce1be23a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/651fce10b05b40a9bb80f173ce1be23a 2024-11-19T08:40:23,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/0e1f6f2c8ae24dc5b7ad6e2734b0c9bd 2024-11-19T08:40:23,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4584a734776b4ac9af20af3485205be9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4584a734776b4ac9af20af3485205be9 2024-11-19T08:40:23,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/c0d08c6d782647159b34f1d2b94b0cc2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/c0d08c6d782647159b34f1d2b94b0cc2 2024-11-19T08:40:23,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/23e6cd9a61ec49dfadba6afcbe71a37b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/23e6cd9a61ec49dfadba6afcbe71a37b 2024-11-19T08:40:23,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/a7328f263fcc40af8b890935929c62ad to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/a7328f263fcc40af8b890935929c62ad 2024-11-19T08:40:23,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8909d83d5912486b8b5fc8e60b7f89c7 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8909d83d5912486b8b5fc8e60b7f89c7 2024-11-19T08:40:23,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f72aa06b28aa4f5a88fc55eb5ab97d92 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f72aa06b28aa4f5a88fc55eb5ab97d92 2024-11-19T08:40:23,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4bdb9bae83e54e87bb796632ccf9f045 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4bdb9bae83e54e87bb796632ccf9f045 2024-11-19T08:40:23,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ca642edb35d141899ef984cedc9df658 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ca642edb35d141899ef984cedc9df658 2024-11-19T08:40:23,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f034ac327dc64bcbb17eba8efefae9ae to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f034ac327dc64bcbb17eba8efefae9ae 2024-11-19T08:40:23,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/893b94e842e54ccfa36dc7a33a581b36 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/893b94e842e54ccfa36dc7a33a581b36 2024-11-19T08:40:23,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ad981635795b4c2c9d57397dd77832ac to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ad981635795b4c2c9d57397dd77832ac 2024-11-19T08:40:23,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/b0cee236d9a648d88568b0326e7c875d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/b0cee236d9a648d88568b0326e7c875d 2024-11-19T08:40:23,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/74f48243564c444cab4b62333f055bc9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/74f48243564c444cab4b62333f055bc9 2024-11-19T08:40:23,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/cac3efde9a184efbb2d588de1ed0fe3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/cac3efde9a184efbb2d588de1ed0fe3e 2024-11-19T08:40:23,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/787d00a4bf1f474ab3455fed3a0ccc5b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/787d00a4bf1f474ab3455fed3a0ccc5b 2024-11-19T08:40:23,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4a140dedfc3649dea592d9088f63eb23 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/4a140dedfc3649dea592d9088f63eb23 2024-11-19T08:40:23,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f280c0d3c11d43b18841f6093483ebf7 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/f280c0d3c11d43b18841f6093483ebf7 2024-11-19T08:40:23,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8085de511a4449f6b4b6986617950735 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/8085de511a4449f6b4b6986617950735 2024-11-19T08:40:23,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ac316546576f49578d21105aafcf60a1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ac316546576f49578d21105aafcf60a1 2024-11-19T08:40:23,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7bf94db8507a4219abb256d2ba956be8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/7bf94db8507a4219abb256d2ba956be8 2024-11-19T08:40:23,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5ac9029677a5403793d6a5c73d6def4a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3232330569014e058d5cb396f0fb90d8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/ef637f5e92524c549e7e7f0119c2fe0f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e5aec7b860d4cb19caaad1fa7ecf68c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bbf9f8efd314fa9ba3850ae92d38997, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c175616d1694ef1a57ef03d8461a40e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/9ad9bc6a7cba4505a10755d86d55d21d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fab9679a2144ba189e5d17c2644fd3c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3d46ad1ce8ed4a2dbaeffd6070ed374f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c570a2bda74840cfacd2a14721454c12, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fbe11c2fb2c4cfeb7fc5c0bad698539, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/2d3555ba22344463a397bc991390519f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/e701468631bf4825aaece86a1543b9d5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5112ebe3acae497aabeffc9979397fe2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5e586ce1ce554f01a854dc16a97ba9ab, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/28e59db38a5e4a488d7bf84f2849a5b0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a3bd5734b28e47f886fd68d58cd0784a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c1170a5e98b34d51b3284a01717ebae5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/16f0c5f99e6048c4a8995076712f05e2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/01530c9dfa4941859a37ceb194615f6e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/062e5d15da7a47b4bfce044b1e9242b4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/54715b8511db4209afb04d3ce8470e5d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bb9506d87154c54bb37502322f97af2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/0f4b1d3b890b406eb7d5bb5e32dcf966, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/b7793e3793024479942fccb930d5ac1f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7c52acd65fef4aa1aa4dd574f396e0ef, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a48e78c0b8564f85a6e5eb9a14050b65] to archive 2024-11-19T08:40:23,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:40:23,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5ac9029677a5403793d6a5c73d6def4a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5ac9029677a5403793d6a5c73d6def4a 2024-11-19T08:40:23,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3232330569014e058d5cb396f0fb90d8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3232330569014e058d5cb396f0fb90d8 2024-11-19T08:40:23,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/ef637f5e92524c549e7e7f0119c2fe0f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/ef637f5e92524c549e7e7f0119c2fe0f 2024-11-19T08:40:23,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e5aec7b860d4cb19caaad1fa7ecf68c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e5aec7b860d4cb19caaad1fa7ecf68c 2024-11-19T08:40:23,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bbf9f8efd314fa9ba3850ae92d38997 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bbf9f8efd314fa9ba3850ae92d38997 2024-11-19T08:40:23,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c175616d1694ef1a57ef03d8461a40e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c175616d1694ef1a57ef03d8461a40e 2024-11-19T08:40:23,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/9ad9bc6a7cba4505a10755d86d55d21d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/9ad9bc6a7cba4505a10755d86d55d21d 2024-11-19T08:40:23,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fab9679a2144ba189e5d17c2644fd3c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fab9679a2144ba189e5d17c2644fd3c 2024-11-19T08:40:23,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3d46ad1ce8ed4a2dbaeffd6070ed374f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/3d46ad1ce8ed4a2dbaeffd6070ed374f 2024-11-19T08:40:23,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c570a2bda74840cfacd2a14721454c12 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c570a2bda74840cfacd2a14721454c12 2024-11-19T08:40:23,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fbe11c2fb2c4cfeb7fc5c0bad698539 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/6fbe11c2fb2c4cfeb7fc5c0bad698539 2024-11-19T08:40:23,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/2d3555ba22344463a397bc991390519f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/2d3555ba22344463a397bc991390519f 2024-11-19T08:40:23,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/e701468631bf4825aaece86a1543b9d5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/e701468631bf4825aaece86a1543b9d5 2024-11-19T08:40:23,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5112ebe3acae497aabeffc9979397fe2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5112ebe3acae497aabeffc9979397fe2 2024-11-19T08:40:23,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5e586ce1ce554f01a854dc16a97ba9ab to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/5e586ce1ce554f01a854dc16a97ba9ab 2024-11-19T08:40:23,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/28e59db38a5e4a488d7bf84f2849a5b0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/28e59db38a5e4a488d7bf84f2849a5b0 2024-11-19T08:40:23,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a3bd5734b28e47f886fd68d58cd0784a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a3bd5734b28e47f886fd68d58cd0784a 2024-11-19T08:40:23,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c1170a5e98b34d51b3284a01717ebae5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c1170a5e98b34d51b3284a01717ebae5 2024-11-19T08:40:23,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/16f0c5f99e6048c4a8995076712f05e2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/16f0c5f99e6048c4a8995076712f05e2 2024-11-19T08:40:23,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/01530c9dfa4941859a37ceb194615f6e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/01530c9dfa4941859a37ceb194615f6e 2024-11-19T08:40:23,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/062e5d15da7a47b4bfce044b1e9242b4 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/062e5d15da7a47b4bfce044b1e9242b4 2024-11-19T08:40:23,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/54715b8511db4209afb04d3ce8470e5d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/54715b8511db4209afb04d3ce8470e5d 2024-11-19T08:40:23,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bb9506d87154c54bb37502322f97af2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8bb9506d87154c54bb37502322f97af2 2024-11-19T08:40:23,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/0f4b1d3b890b406eb7d5bb5e32dcf966 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/0f4b1d3b890b406eb7d5bb5e32dcf966 2024-11-19T08:40:23,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/b7793e3793024479942fccb930d5ac1f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/b7793e3793024479942fccb930d5ac1f 2024-11-19T08:40:23,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7c52acd65fef4aa1aa4dd574f396e0ef to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7c52acd65fef4aa1aa4dd574f396e0ef 2024-11-19T08:40:23,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a48e78c0b8564f85a6e5eb9a14050b65 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/a48e78c0b8564f85a6e5eb9a14050b65 2024-11-19T08:40:23,967 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-19T08:40:23,970 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. 2024-11-19T08:40:23,970 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for ca9bd3485db2e8c855878994330f02da: Waiting for close lock at 1732005622041Running coprocessor pre-close hooks at 1732005622041Disabling compacts and flushes for region at 1732005622041Disabling writes for close at 1732005622448 (+407 ms)Obtaining lock to block concurrent updates at 1732005622449 (+1 ms)Preparing flush snapshotting stores in ca9bd3485db2e8c855878994330f02da at 1732005622449Finished memstore snapshotting TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732005622449Flushing stores of TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da. at 1732005622450 (+1 ms)Flushing ca9bd3485db2e8c855878994330f02da/A: creating writer at 1732005622450Flushing ca9bd3485db2e8c855878994330f02da/A: appending metadata at 1732005622453 (+3 ms)Flushing ca9bd3485db2e8c855878994330f02da/A: closing flushed file at 1732005622453Flushing ca9bd3485db2e8c855878994330f02da/B: creating writer at 1732005622864 (+411 ms)Flushing ca9bd3485db2e8c855878994330f02da/B: appending metadata at 1732005622866 (+2 ms)Flushing ca9bd3485db2e8c855878994330f02da/B: closing flushed file at 1732005622867 (+1 ms)Flushing ca9bd3485db2e8c855878994330f02da/C: creating writer at 1732005623278 (+411 ms)Flushing ca9bd3485db2e8c855878994330f02da/C: appending metadata at 1732005623281 (+3 ms)Flushing ca9bd3485db2e8c855878994330f02da/C: closing flushed file at 1732005623281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@444d8961: reopening flushed file at 1732005623694 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b434602: reopening flushed file at 1732005623712 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d166cc5: reopening flushed file at 1732005623721 (+9 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for ca9bd3485db2e8c855878994330f02da in 1281ms, sequenceid=495, compaction requested=true at 1732005623730 (+9 ms)Writing region close event to WAL at 1732005623962 (+232 ms)Running coprocessor post-close hooks at 1732005623968 (+6 ms)Closed at 1732005623970 (+2 ms) 2024-11-19T08:40:23,977 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:23,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ca9bd3485db2e8c855878994330f02da, regionState=CLOSED 2024-11-19T08:40:23,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:40:23,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T08:40:23,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure ca9bd3485db2e8c855878994330f02da, server=3ab37fa97a98,34165,1732005593269 in 2.1020 sec 2024-11-19T08:40:23,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-19T08:40:23,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca9bd3485db2e8c855878994330f02da, UNASSIGN in 2.1150 sec 2024-11-19T08:40:24,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T08:40:24,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1270 sec 2024-11-19T08:40:24,010 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005624010"}]},"ts":"1732005624010"} 2024-11-19T08:40:24,012 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T08:40:24,013 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T08:40:24,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1500 sec 2024-11-19T08:40:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T08:40:24,027 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:24,030 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T08:40:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:24,036 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T08:40:24,038 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:24,046 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:24,053 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/recovered.edits] 2024-11-19T08:40:24,057 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/343d985553c742ea9e648373c54a3dbf to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/343d985553c742ea9e648373c54a3dbf 2024-11-19T08:40:24,058 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/4bf15cf1d8514c77ab0d358dde35d577 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/4bf15cf1d8514c77ab0d358dde35d577 2024-11-19T08:40:24,060 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/763f64ede0a04bdfb84b41058ecce504 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/763f64ede0a04bdfb84b41058ecce504 2024-11-19T08:40:24,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/76ddb4f26a0d434fb0c5de31eb04ad33 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/76ddb4f26a0d434fb0c5de31eb04ad33 2024-11-19T08:40:24,063 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9312c7c3155840818344f44f31a65450 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/9312c7c3155840818344f44f31a65450 2024-11-19T08:40:24,065 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/999ee461afb9491082b6f0224c1ba288 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/999ee461afb9491082b6f0224c1ba288 2024-11-19T08:40:24,066 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7ccfd61188745578e8d0505f178802f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/A/c7ccfd61188745578e8d0505f178802f 2024-11-19T08:40:24,069 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/870e17c87916448eb1c27d08bb1bedcf to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/870e17c87916448eb1c27d08bb1bedcf 2024-11-19T08:40:24,071 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/92e182cabe984cc580cd4e2338c0e11d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/92e182cabe984cc580cd4e2338c0e11d 2024-11-19T08:40:24,073 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/949a4b50f92c46f1bac8cfe086ef728b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/949a4b50f92c46f1bac8cfe086ef728b 2024-11-19T08:40:24,074 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d19e255f385c4391846e7369f499f2f3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d19e255f385c4391846e7369f499f2f3 2024-11-19T08:40:24,076 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d42cb46fe7444cde8a6285d8f24e2005 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/d42cb46fe7444cde8a6285d8f24e2005 2024-11-19T08:40:24,077 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ec68013e02cb43da8876fd61e0590613 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ec68013e02cb43da8876fd61e0590613 2024-11-19T08:40:24,080 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ff3ab1b60b0f44f181d64622db61e1fe to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/B/ff3ab1b60b0f44f181d64622db61e1fe 2024-11-19T08:40:24,083 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c57b159c6b6458996b373b83f3ef541 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/1c57b159c6b6458996b373b83f3ef541 2024-11-19T08:40:24,085 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4acf227bdbb1461da0d8b00c0f0a9189 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4acf227bdbb1461da0d8b00c0f0a9189 2024-11-19T08:40:24,087 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4dbf525b5053461397a1a1c2d8f05e1f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/4dbf525b5053461397a1a1c2d8f05e1f 2024-11-19T08:40:24,088 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e21ba1439cd42fc8d1d3f2034464505 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/7e21ba1439cd42fc8d1d3f2034464505 2024-11-19T08:40:24,090 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8083deeafd4b4ff596fa5a7b3f3bd359 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8083deeafd4b4ff596fa5a7b3f3bd359 2024-11-19T08:40:24,093 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8811189c6a9b4e75ada6ca5c46e5e4c1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/8811189c6a9b4e75ada6ca5c46e5e4c1 2024-11-19T08:40:24,101 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c60389ea1d524d5c89f2e381d8e77464 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/C/c60389ea1d524d5c89f2e381d8e77464 2024-11-19T08:40:24,107 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/recovered.edits/498.seqid to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da/recovered.edits/498.seqid 2024-11-19T08:40:24,108 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/ca9bd3485db2e8c855878994330f02da 2024-11-19T08:40:24,108 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T08:40:24,114 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:24,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T08:40:24,127 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T08:40:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T08:40:24,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T08:40:24,170 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:24,170 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T08:40:24,170 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732005624170"}]},"ts":"9223372036854775807"} 2024-11-19T08:40:24,175 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T08:40:24,175 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ca9bd3485db2e8c855878994330f02da, NAME => 'TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T08:40:24,175 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T08:40:24,175 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732005624175"}]},"ts":"9223372036854775807"} 2024-11-19T08:40:24,178 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T08:40:24,179 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:24,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 149 msec 2024-11-19T08:40:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T08:40:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T08:40:24,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:24,368 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1607800269_22 at /127.0.0.1:59540 [Waiting for operation #103] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1816896270_22 at /127.0.0.1:42570 [Waiting for operation #275] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1816896270_22 at /127.0.0.1:42694 [Waiting for operation #255] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;3ab37fa97a98:34165-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 299) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6510 (was 6513) 2024-11-19T08:40:24,377 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=6509 2024-11-19T08:40:24,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:40:24,379 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:40:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:24,381 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T08:40:24,381 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:24,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-19T08:40:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T08:40:24,382 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T08:40:24,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741939_1115 (size=959) 2024-11-19T08:40:24,398 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:40:24,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741940_1116 (size=53) 2024-11-19T08:40:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T08:40:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T08:40:24,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:40:24,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 392ed962c42d18afe3f39ea7d3a6fd3e, disabling compactions & flushes 2024-11-19T08:40:24,805 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:24,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:24,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. after waiting 0 ms 2024-11-19T08:40:24,806 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:24,806 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:24,806 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 392ed962c42d18afe3f39ea7d3a6fd3e: Waiting for close lock at 1732005624805Disabling compacts and flushes for region at 1732005624805Disabling writes for close at 1732005624806 (+1 ms)Writing region close event to WAL at 1732005624806Closed at 1732005624806 2024-11-19T08:40:24,807 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T08:40:24,808 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732005624807"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732005624807"}]},"ts":"1732005624807"} 2024-11-19T08:40:24,810 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T08:40:24,811 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T08:40:24,812 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005624811"}]},"ts":"1732005624811"} 2024-11-19T08:40:24,814 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T08:40:24,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, ASSIGN}] 2024-11-19T08:40:24,815 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, ASSIGN 2024-11-19T08:40:24,816 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, ASSIGN; state=OFFLINE, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=false 2024-11-19T08:40:24,967 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:24,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, ASSIGN because future has completed 2024-11-19T08:40:24,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:40:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T08:40:25,125 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:25,126 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:40:25,126 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,126 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:40:25,126 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,126 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,128 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,130 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 392ed962c42d18afe3f39ea7d3a6fd3e columnFamilyName A 2024-11-19T08:40:25,131 DEBUG [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:25,131 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(327): Store=392ed962c42d18afe3f39ea7d3a6fd3e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:25,132 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,133 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 392ed962c42d18afe3f39ea7d3a6fd3e columnFamilyName B 2024-11-19T08:40:25,133 DEBUG [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:25,134 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(327): Store=392ed962c42d18afe3f39ea7d3a6fd3e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:25,134 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,135 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 392ed962c42d18afe3f39ea7d3a6fd3e columnFamilyName C 2024-11-19T08:40:25,135 DEBUG [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:25,136 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(327): Store=392ed962c42d18afe3f39ea7d3a6fd3e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:25,136 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,137 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,137 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,138 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,138 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,139 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:40:25,140 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,142 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:40:25,143 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 392ed962c42d18afe3f39ea7d3a6fd3e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70658461, jitterRate=0.05289311707019806}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:40:25,143 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,144 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 392ed962c42d18afe3f39ea7d3a6fd3e: Running coprocessor pre-open hook at 1732005625126Writing region info on filesystem at 1732005625126Initializing all the Stores at 1732005625127 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005625127Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005625128 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005625128Cleaning up temporary data from old regions at 1732005625138 (+10 ms)Running coprocessor post-open hooks at 1732005625143 (+5 ms)Region opened successfully at 1732005625144 (+1 ms) 2024-11-19T08:40:25,145 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., pid=34, masterSystemTime=1732005625121 2024-11-19T08:40:25,148 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:25,148 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:25,148 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=OPEN, openSeqNum=2, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:25,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:40:25,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T08:40:25,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 in 182 msec 2024-11-19T08:40:25,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T08:40:25,156 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, ASSIGN in 340 msec 2024-11-19T08:40:25,157 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T08:40:25,157 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005625157"}]},"ts":"1732005625157"} 2024-11-19T08:40:25,159 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T08:40:25,160 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T08:40:25,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 781 msec 2024-11-19T08:40:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T08:40:25,516 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:25,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:25,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:25,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:25,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c45a5be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:25,522 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f1990d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:25,523 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:25,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:25,527 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T08:40:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ab37fa97a98,36325,1732005592414 2024-11-19T08:40:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@695273e8 2024-11-19T08:40:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T08:40:25,530 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T08:40:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:40:25,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:40:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:25,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741941_1117 (size=995) 2024-11-19T08:40:25,563 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T08:40:25,563 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T08:40:25,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:40:25,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, REOPEN/MOVE}] 2024-11-19T08:40:25,582 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, REOPEN/MOVE 2024-11-19T08:40:25,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:25,585 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36325 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=3ab37fa97a98,34165,1732005593269, table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T08:40:25,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, REOPEN/MOVE because future has completed 2024-11-19T08:40:25,596 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:40:25,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:40:25,750 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,751 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:40:25,751 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 392ed962c42d18afe3f39ea7d3a6fd3e, disabling compactions & flushes 2024-11-19T08:40:25,751 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:25,751 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:25,751 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. after waiting 0 ms 2024-11-19T08:40:25,751 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:25,755 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T08:40:25,755 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:25,756 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 392ed962c42d18afe3f39ea7d3a6fd3e: Waiting for close lock at 1732005625751Running coprocessor pre-close hooks at 1732005625751Disabling compacts and flushes for region at 1732005625751Disabling writes for close at 1732005625751Writing region close event to WAL at 1732005625751Running coprocessor post-close hooks at 1732005625755 (+4 ms)Closed at 1732005625755 2024-11-19T08:40:25,756 WARN [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 392ed962c42d18afe3f39ea7d3a6fd3e to self. 2024-11-19T08:40:25,758 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:25,758 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=CLOSED 2024-11-19T08:40:25,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:40:25,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-19T08:40:25,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 in 166 msec 2024-11-19T08:40:25,765 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, REOPEN/MOVE; state=CLOSED, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=true 2024-11-19T08:40:25,916 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:25,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, REOPEN/MOVE because future has completed 2024-11-19T08:40:25,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:40:26,072 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,073 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:40:26,073 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,073 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:40:26,073 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,073 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,078 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,084 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 392ed962c42d18afe3f39ea7d3a6fd3e columnFamilyName A 2024-11-19T08:40:26,087 DEBUG [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:26,087 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(327): Store=392ed962c42d18afe3f39ea7d3a6fd3e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:26,088 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,089 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 392ed962c42d18afe3f39ea7d3a6fd3e columnFamilyName B 2024-11-19T08:40:26,089 DEBUG [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:26,089 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(327): Store=392ed962c42d18afe3f39ea7d3a6fd3e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:26,089 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,090 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 392ed962c42d18afe3f39ea7d3a6fd3e columnFamilyName C 2024-11-19T08:40:26,090 DEBUG [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:26,090 INFO [StoreOpener-392ed962c42d18afe3f39ea7d3a6fd3e-1 {}] regionserver.HStore(327): Store=392ed962c42d18afe3f39ea7d3a6fd3e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:26,091 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,091 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,092 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,093 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,093 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,094 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:40:26,095 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,096 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 392ed962c42d18afe3f39ea7d3a6fd3e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74313628, jitterRate=0.10735934972763062}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:40:26,096 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,097 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 392ed962c42d18afe3f39ea7d3a6fd3e: Running coprocessor pre-open hook at 1732005626073Writing region info on filesystem at 1732005626073Initializing all the Stores at 1732005626074 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005626074Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005626075 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005626075Cleaning up temporary data from old regions at 1732005626093 (+18 ms)Running coprocessor post-open hooks at 1732005626096 (+3 ms)Region opened successfully at 1732005626097 (+1 ms) 2024-11-19T08:40:26,097 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., pid=39, masterSystemTime=1732005626069 2024-11-19T08:40:26,099 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,100 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,100 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=OPEN, openSeqNum=5, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:40:26,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-19T08:40:26,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 in 185 msec 2024-11-19T08:40:26,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T08:40:26,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, REOPEN/MOVE in 526 msec 2024-11-19T08:40:26,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T08:40:26,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 539 msec 2024-11-19T08:40:26,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 575 msec 2024-11-19T08:40:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-19T08:40:26,119 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:26,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,121 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6036f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,123 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f321aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,125 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,128 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232574cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,130 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e0d0dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,132 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7c6002, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,135 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f9d34e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,137 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563e71dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,139 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7e83b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,141 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736fa1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,144 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d721c65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,146 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ceddc36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,148 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa6b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,150 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a6519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,152 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d661940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,154 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739a3af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:26,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:26,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a8cc242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:26,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:26,160 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,161 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,161 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,164 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,164 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T08:40:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,166 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,166 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,167 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:26,167 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T08:40:26,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,170 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:26,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,178 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:26,180 DEBUG [Thread-612 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:26,180 DEBUG [Thread-612 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:26,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:40:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:26,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:40:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T08:40:26,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111947c4dc9f2f9d4342ade70a965916c77f_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005626179/Put/seqid=0 2024-11-19T08:40:26,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005686285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005686294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005686295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005686296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005686300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:26,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:26,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741942_1118 (size=12154) 2024-11-19T08:40:26,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:26,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111947c4dc9f2f9d4342ade70a965916c77f_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111947c4dc9f2f9d4342ade70a965916c77f_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,353 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/88a6d2a28ae748689793477c1f421ad5, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:26,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/88a6d2a28ae748689793477c1f421ad5 is 175, key is test_row_0/A:col10/1732005626179/Put/seqid=0 2024-11-19T08:40:26,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741943_1119 (size=30955) 2024-11-19T08:40:26,392 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/88a6d2a28ae748689793477c1f421ad5 2024-11-19T08:40:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005686407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005686407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005686407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005686417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005686418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/160297109afc46efa50b404adefae3c3 is 50, key is test_row_0/B:col10/1732005626179/Put/seqid=0 2024-11-19T08:40:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741944_1120 (size=12001) 2024-11-19T08:40:26,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/160297109afc46efa50b404adefae3c3 2024-11-19T08:40:26,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:26,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:26,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T08:40:26,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a6ae602b58df47ab990a5f7111bf4199 is 50, key is test_row_0/C:col10/1732005626179/Put/seqid=0 2024-11-19T08:40:26,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741945_1121 (size=12001) 2024-11-19T08:40:26,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a6ae602b58df47ab990a5f7111bf4199 2024-11-19T08:40:26,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/88a6d2a28ae748689793477c1f421ad5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/88a6d2a28ae748689793477c1f421ad5 2024-11-19T08:40:26,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/88a6d2a28ae748689793477c1f421ad5, entries=150, sequenceid=15, filesize=30.2 K 2024-11-19T08:40:26,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/160297109afc46efa50b404adefae3c3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/160297109afc46efa50b404adefae3c3 2024-11-19T08:40:26,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/160297109afc46efa50b404adefae3c3, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T08:40:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a6ae602b58df47ab990a5f7111bf4199 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a6ae602b58df47ab990a5f7111bf4199 2024-11-19T08:40:26,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a6ae602b58df47ab990a5f7111bf4199, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T08:40:26,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 392ed962c42d18afe3f39ea7d3a6fd3e in 377ms, sequenceid=15, compaction requested=false 2024-11-19T08:40:26,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:26,624 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:26,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T08:40:26,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:26,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:26,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cce00abf3af340ea87ff41ac3a341fb9_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005626273/Put/seqid=0 2024-11-19T08:40:26,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005686641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005686642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005686645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005686646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005686656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741946_1122 (size=14594) 2024-11-19T08:40:26,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005686767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005686767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005686767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005686776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T08:40:26,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:26,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:26,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:26,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:26,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005686967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005686976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005686976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005686976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:26,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005686986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cce00abf3af340ea87ff41ac3a341fb9_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cce00abf3af340ea87ff41ac3a341fb9_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:27,079 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8d75f7ce2ed74d48838e08a29950e8ea, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:27,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8d75f7ce2ed74d48838e08a29950e8ea is 175, key is test_row_0/A:col10/1732005626273/Put/seqid=0 2024-11-19T08:40:27,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:27,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741947_1123 (size=39549) 2024-11-19T08:40:27,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8d75f7ce2ed74d48838e08a29950e8ea 2024-11-19T08:40:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/9f081ad92047428eb890b3572ac703b0 is 50, key is test_row_0/B:col10/1732005626273/Put/seqid=0 2024-11-19T08:40:27,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741948_1124 (size=12001) 2024-11-19T08:40:27,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/9f081ad92047428eb890b3572ac703b0 2024-11-19T08:40:27,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/b0452af38538462485e5ecd4e790d167 is 50, key is test_row_0/C:col10/1732005626273/Put/seqid=0 2024-11-19T08:40:27,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741949_1125 (size=12001) 2024-11-19T08:40:27,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/b0452af38538462485e5ecd4e790d167 2024-11-19T08:40:27,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8d75f7ce2ed74d48838e08a29950e8ea as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8d75f7ce2ed74d48838e08a29950e8ea 2024-11-19T08:40:27,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8d75f7ce2ed74d48838e08a29950e8ea, entries=200, sequenceid=42, filesize=38.6 K 2024-11-19T08:40:27,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/9f081ad92047428eb890b3572ac703b0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/9f081ad92047428eb890b3572ac703b0 2024-11-19T08:40:27,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/9f081ad92047428eb890b3572ac703b0, entries=150, sequenceid=42, filesize=11.7 K 2024-11-19T08:40:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/b0452af38538462485e5ecd4e790d167 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/b0452af38538462485e5ecd4e790d167 2024-11-19T08:40:27,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/b0452af38538462485e5ecd4e790d167, entries=150, sequenceid=42, filesize=11.7 K 2024-11-19T08:40:27,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 392ed962c42d18afe3f39ea7d3a6fd3e in 642ms, sequenceid=42, compaction requested=false 2024-11-19T08:40:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T08:40:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:27,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:40:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196133564786e245bc8db6a1899c9aa695_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005627307/Put/seqid=0 2024-11-19T08:40:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741950_1126 (size=17034) 2024-11-19T08:40:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,366 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196133564786e245bc8db6a1899c9aa695_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196133564786e245bc8db6a1899c9aa695_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,373 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/92757d2c4e6043409fe4c5ef3b055574, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/92757d2c4e6043409fe4c5ef3b055574 is 175, key is test_row_0/A:col10/1732005627307/Put/seqid=0 2024-11-19T08:40:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741951_1127 (size=48135) 2024-11-19T08:40:27,404 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/92757d2c4e6043409fe4c5ef3b055574 2024-11-19T08:40:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:27,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:27,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/fffc22e07c974f4c9a9eeed306664ef7 is 50, key is test_row_0/B:col10/1732005627307/Put/seqid=0 2024-11-19T08:40:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:27,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005687446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005687449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005687449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005687458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741952_1128 (size=16677) 2024-11-19T08:40:27,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/fffc22e07c974f4c9a9eeed306664ef7 2024-11-19T08:40:27,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005687477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/c8d314b5445740d3ae734443950d89f9 is 50, key is test_row_0/C:col10/1732005627307/Put/seqid=0 2024-11-19T08:40:27,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741953_1129 (size=9657) 2024-11-19T08:40:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005687566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005687566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005687566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005687576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:27,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005687777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005687777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005687786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005687787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:27,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:27,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:27,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:27,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:27,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:27,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:27,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/c8d314b5445740d3ae734443950d89f9 2024-11-19T08:40:27,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/92757d2c4e6043409fe4c5ef3b055574 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/92757d2c4e6043409fe4c5ef3b055574 2024-11-19T08:40:27,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/92757d2c4e6043409fe4c5ef3b055574, entries=250, sequenceid=54, filesize=47.0 K 2024-11-19T08:40:27,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/fffc22e07c974f4c9a9eeed306664ef7 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fffc22e07c974f4c9a9eeed306664ef7 2024-11-19T08:40:27,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fffc22e07c974f4c9a9eeed306664ef7, entries=250, sequenceid=54, filesize=16.3 K 2024-11-19T08:40:28,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/c8d314b5445740d3ae734443950d89f9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/c8d314b5445740d3ae734443950d89f9 2024-11-19T08:40:28,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/c8d314b5445740d3ae734443950d89f9, entries=100, sequenceid=54, filesize=9.4 K 2024-11-19T08:40:28,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 392ed962c42d18afe3f39ea7d3a6fd3e in 700ms, sequenceid=54, compaction requested=true 2024-11-19T08:40:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:28,007 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:28,007 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:28,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:28,009 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:28,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:28,009 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/B is initiating minor compaction (all files) 2024-11-19T08:40:28,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/A is initiating minor compaction (all files) 2024-11-19T08:40:28,009 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/B in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:28,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/A in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:28,009 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/160297109afc46efa50b404adefae3c3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/9f081ad92047428eb890b3572ac703b0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fffc22e07c974f4c9a9eeed306664ef7] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=39.7 K 2024-11-19T08:40:28,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/88a6d2a28ae748689793477c1f421ad5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8d75f7ce2ed74d48838e08a29950e8ea, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/92757d2c4e6043409fe4c5ef3b055574] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=115.9 K 2024-11-19T08:40:28,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:28,010 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/88a6d2a28ae748689793477c1f421ad5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8d75f7ce2ed74d48838e08a29950e8ea, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/92757d2c4e6043409fe4c5ef3b055574] 2024-11-19T08:40:28,010 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 160297109afc46efa50b404adefae3c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732005626179 2024-11-19T08:40:28,011 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f081ad92047428eb890b3572ac703b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732005626273 2024-11-19T08:40:28,017 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a6d2a28ae748689793477c1f421ad5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732005626179 2024-11-19T08:40:28,017 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting fffc22e07c974f4c9a9eeed306664ef7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005626636 2024-11-19T08:40:28,017 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d75f7ce2ed74d48838e08a29950e8ea, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732005626273 2024-11-19T08:40:28,018 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92757d2c4e6043409fe4c5ef3b055574, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005626636 2024-11-19T08:40:28,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T08:40:28,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:28,030 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T08:40:28,036 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:28,049 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#B#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:28,050 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/b94357ee6096417c937054bf4d71a06f is 50, key is test_row_0/B:col10/1732005627307/Put/seqid=0 2024-11-19T08:40:28,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194be281738e2248c59adb92908d9fe19e_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005627406/Put/seqid=0 2024-11-19T08:40:28,064 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198c59a67b7997440f8d00c7f15a8be969_392ed962c42d18afe3f39ea7d3a6fd3e store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:28,070 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198c59a67b7997440f8d00c7f15a8be969_392ed962c42d18afe3f39ea7d3a6fd3e, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:28,071 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198c59a67b7997440f8d00c7f15a8be969_392ed962c42d18afe3f39ea7d3a6fd3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:28,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:28,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741956_1132 (size=4469) 2024-11-19T08:40:28,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741955_1131 (size=12154) 2024-11-19T08:40:28,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005688111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005688114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005688115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005688115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741954_1130 (size=12104) 2024-11-19T08:40:28,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005688226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005688227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005688236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005688236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T08:40:28,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005688436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005688438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005688446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005688446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005688497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,499 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 392ed962c42d18afe3f39ea7d3a6fd3e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:28,517 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#A#compaction#111 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:28,517 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/c9df7e51ccbc4c3c8ab36aaa52711b9c is 175, key is test_row_0/A:col10/1732005627307/Put/seqid=0 2024-11-19T08:40:28,531 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194be281738e2248c59adb92908d9fe19e_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be281738e2248c59adb92908d9fe19e_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f730f0dedffb4d028b2c76b5d48cb91a, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f730f0dedffb4d028b2c76b5d48cb91a is 175, key is test_row_0/A:col10/1732005627406/Put/seqid=0 2024-11-19T08:40:28,549 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/b94357ee6096417c937054bf4d71a06f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b94357ee6096417c937054bf4d71a06f 2024-11-19T08:40:28,559 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/B of 392ed962c42d18afe3f39ea7d3a6fd3e into b94357ee6096417c937054bf4d71a06f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:28,559 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:28,559 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/B, priority=13, startTime=1732005628007; duration=0sec 2024-11-19T08:40:28,559 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:28,559 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:B 2024-11-19T08:40:28,559 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:28,563 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:28,563 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/C is initiating minor compaction (all files) 2024-11-19T08:40:28,563 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/C in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:28,563 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a6ae602b58df47ab990a5f7111bf4199, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/b0452af38538462485e5ecd4e790d167, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/c8d314b5445740d3ae734443950d89f9] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=32.9 K 2024-11-19T08:40:28,564 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ae602b58df47ab990a5f7111bf4199, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732005626179 2024-11-19T08:40:28,564 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b0452af38538462485e5ecd4e790d167, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732005626273 2024-11-19T08:40:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741957_1133 (size=31165) 2024-11-19T08:40:28,565 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c8d314b5445740d3ae734443950d89f9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005627303 2024-11-19T08:40:28,576 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/c9df7e51ccbc4c3c8ab36aaa52711b9c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c9df7e51ccbc4c3c8ab36aaa52711b9c 2024-11-19T08:40:28,586 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/A of 392ed962c42d18afe3f39ea7d3a6fd3e into c9df7e51ccbc4c3c8ab36aaa52711b9c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:28,586 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:28,586 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/A, priority=13, startTime=1732005628007; duration=0sec 2024-11-19T08:40:28,586 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:28,586 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:A 2024-11-19T08:40:28,588 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:28,589 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/66717e6c1548471ebc64bac04a31c0e3 is 50, key is test_row_0/C:col10/1732005627307/Put/seqid=0 2024-11-19T08:40:28,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741958_1134 (size=30955) 2024-11-19T08:40:28,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f730f0dedffb4d028b2c76b5d48cb91a 2024-11-19T08:40:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/0e63e59b987340fb9c3ff16851b99756 is 50, key is test_row_0/B:col10/1732005627406/Put/seqid=0 2024-11-19T08:40:28,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741959_1135 (size=12104) 2024-11-19T08:40:28,624 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/66717e6c1548471ebc64bac04a31c0e3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/66717e6c1548471ebc64bac04a31c0e3 2024-11-19T08:40:28,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741960_1136 (size=12001) 2024-11-19T08:40:28,636 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/C of 392ed962c42d18afe3f39ea7d3a6fd3e into 66717e6c1548471ebc64bac04a31c0e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:28,636 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:28,636 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/C, priority=13, startTime=1732005628007; duration=0sec 2024-11-19T08:40:28,636 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:28,636 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:C 2024-11-19T08:40:28,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005688746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005688756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005688756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005688756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:28,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,030 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/0e63e59b987340fb9c3ff16851b99756 2024-11-19T08:40:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/ca0e736a179941d4a897538a38d64457 is 50, key is test_row_0/C:col10/1732005627406/Put/seqid=0 2024-11-19T08:40:29,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741961_1137 (size=12001) 2024-11-19T08:40:29,069 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/ca0e736a179941d4a897538a38d64457 2024-11-19T08:40:29,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f730f0dedffb4d028b2c76b5d48cb91a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f730f0dedffb4d028b2c76b5d48cb91a 2024-11-19T08:40:29,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f730f0dedffb4d028b2c76b5d48cb91a, entries=150, sequenceid=78, filesize=30.2 K 2024-11-19T08:40:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/0e63e59b987340fb9c3ff16851b99756 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/0e63e59b987340fb9c3ff16851b99756 2024-11-19T08:40:29,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/0e63e59b987340fb9c3ff16851b99756, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T08:40:29,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/ca0e736a179941d4a897538a38d64457 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/ca0e736a179941d4a897538a38d64457 2024-11-19T08:40:29,106 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/ca0e736a179941d4a897538a38d64457, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T08:40:29,107 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1078ms, sequenceid=78, compaction requested=false 2024-11-19T08:40:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T08:40:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T08:40:29,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T08:40:29,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9390 sec 2024-11-19T08:40:29,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.9520 sec 2024-11-19T08:40:29,260 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:29,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T08:40:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:29,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d43e0895a554622833c0dd60edfabb9_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005628111/Put/seqid=0 2024-11-19T08:40:29,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741962_1138 (size=14594) 2024-11-19T08:40:29,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005689344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005689352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005689354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005689356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005689467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005689467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005689467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005689467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,505 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T08:40:29,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005689676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005689676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005689676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005689676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,745 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:29,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198d43e0895a554622833c0dd60edfabb9_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d43e0895a554622833c0dd60edfabb9_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:29,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/7dd798fe465a41c4a335d084d34de998, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:29,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/7dd798fe465a41c4a335d084d34de998 is 175, key is test_row_0/A:col10/1732005628111/Put/seqid=0 2024-11-19T08:40:29,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741963_1139 (size=39549) 2024-11-19T08:40:29,787 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/7dd798fe465a41c4a335d084d34de998 2024-11-19T08:40:29,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/13cc135fd6ca4190aa7a4544325d1691 is 50, key is test_row_0/B:col10/1732005628111/Put/seqid=0 2024-11-19T08:40:29,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741964_1140 (size=12001) 2024-11-19T08:40:29,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005689986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:30,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/13cc135fd6ca4190aa7a4544325d1691 2024-11-19T08:40:30,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/f34796759b6b49d091d540ea8dbe0fa5 is 50, key is test_row_0/C:col10/1732005628111/Put/seqid=0 2024-11-19T08:40:30,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741965_1141 (size=12001) 2024-11-19T08:40:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T08:40:30,347 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:30,347 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T08:40:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T08:40:30,349 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:30,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:30,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T08:40:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005690496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:30,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005690496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005690497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:30,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T08:40:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:30,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005690506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005690526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:30,530 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 392ed962c42d18afe3f39ea7d3a6fd3e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:30,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T08:40:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:30,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T08:40:30,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/f34796759b6b49d091d540ea8dbe0fa5 2024-11-19T08:40:30,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/7dd798fe465a41c4a335d084d34de998 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/7dd798fe465a41c4a335d084d34de998 2024-11-19T08:40:30,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/7dd798fe465a41c4a335d084d34de998, entries=200, sequenceid=95, filesize=38.6 K 2024-11-19T08:40:30,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/13cc135fd6ca4190aa7a4544325d1691 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/13cc135fd6ca4190aa7a4544325d1691 2024-11-19T08:40:30,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/13cc135fd6ca4190aa7a4544325d1691, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T08:40:30,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/f34796759b6b49d091d540ea8dbe0fa5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/f34796759b6b49d091d540ea8dbe0fa5 2024-11-19T08:40:30,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/f34796759b6b49d091d540ea8dbe0fa5, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T08:40:30,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1446ms, sequenceid=95, compaction requested=true 2024-11-19T08:40:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:30,706 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T08:40:30,706 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:30,708 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:30,708 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/A is initiating minor compaction (all files) 2024-11-19T08:40:30,708 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/A in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,708 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c9df7e51ccbc4c3c8ab36aaa52711b9c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f730f0dedffb4d028b2c76b5d48cb91a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/7dd798fe465a41c4a335d084d34de998] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=99.3 K 2024-11-19T08:40:30,708 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,708 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c9df7e51ccbc4c3c8ab36aaa52711b9c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f730f0dedffb4d028b2c76b5d48cb91a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/7dd798fe465a41c4a335d084d34de998] 2024-11-19T08:40:30,709 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9df7e51ccbc4c3c8ab36aaa52711b9c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005626294 2024-11-19T08:40:30,710 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:30,710 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/B is initiating minor compaction (all files) 2024-11-19T08:40:30,710 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/B in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,710 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b94357ee6096417c937054bf4d71a06f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/0e63e59b987340fb9c3ff16851b99756, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/13cc135fd6ca4190aa7a4544325d1691] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=35.3 K 2024-11-19T08:40:30,710 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f730f0dedffb4d028b2c76b5d48cb91a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732005627406 2024-11-19T08:40:30,710 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b94357ee6096417c937054bf4d71a06f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005626294 2024-11-19T08:40:30,711 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e63e59b987340fb9c3ff16851b99756, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732005627406 2024-11-19T08:40:30,711 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dd798fe465a41c4a335d084d34de998, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005628102 2024-11-19T08:40:30,711 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 13cc135fd6ca4190aa7a4544325d1691, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005628110 2024-11-19T08:40:30,733 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#B#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:30,734 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/634e7f7ecfad4e64bcd8588636184b51 is 50, key is test_row_0/B:col10/1732005628111/Put/seqid=0 2024-11-19T08:40:30,739 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:30,751 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194ec84e571b1041d7806e8d9caaccc0c4_392ed962c42d18afe3f39ea7d3a6fd3e store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:30,754 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194ec84e571b1041d7806e8d9caaccc0c4_392ed962c42d18afe3f39ea7d3a6fd3e, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:30,755 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194ec84e571b1041d7806e8d9caaccc0c4_392ed962c42d18afe3f39ea7d3a6fd3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:30,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741966_1142 (size=12207) 2024-11-19T08:40:30,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T08:40:30,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,812 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T08:40:30,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741967_1143 (size=4469) 2024-11-19T08:40:30,815 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#A#compaction#121 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:30,815 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/9a67e94053c4431ebf0b20cd510bfed5 is 175, key is test_row_0/A:col10/1732005628111/Put/seqid=0 2024-11-19T08:40:30,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119be47999c00634f0cb905fb9c4194d745_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005629349/Put/seqid=0 2024-11-19T08:40:30,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741968_1144 (size=31161) 2024-11-19T08:40:30,887 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/9a67e94053c4431ebf0b20cd510bfed5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/9a67e94053c4431ebf0b20cd510bfed5 2024-11-19T08:40:30,895 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/A of 392ed962c42d18afe3f39ea7d3a6fd3e into 9a67e94053c4431ebf0b20cd510bfed5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:30,895 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:30,895 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/A, priority=13, startTime=1732005630706; duration=0sec 2024-11-19T08:40:30,895 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:30,895 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:A 2024-11-19T08:40:30,895 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:30,896 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:30,897 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/C is initiating minor compaction (all files) 2024-11-19T08:40:30,897 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/C in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:30,897 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/66717e6c1548471ebc64bac04a31c0e3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/ca0e736a179941d4a897538a38d64457, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/f34796759b6b49d091d540ea8dbe0fa5] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=35.3 K 2024-11-19T08:40:30,897 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66717e6c1548471ebc64bac04a31c0e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005626294 2024-11-19T08:40:30,898 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0e736a179941d4a897538a38d64457, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732005627406 2024-11-19T08:40:30,898 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f34796759b6b49d091d540ea8dbe0fa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005628110 2024-11-19T08:40:30,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741969_1145 (size=12154) 2024-11-19T08:40:30,917 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:30,918 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/89d3285dae62450a99c90668c6fbfa31 is 50, key is test_row_0/C:col10/1732005628111/Put/seqid=0 2024-11-19T08:40:30,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741970_1146 (size=12207) 2024-11-19T08:40:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T08:40:31,185 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/634e7f7ecfad4e64bcd8588636184b51 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/634e7f7ecfad4e64bcd8588636184b51 2024-11-19T08:40:31,194 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/B of 392ed962c42d18afe3f39ea7d3a6fd3e into 634e7f7ecfad4e64bcd8588636184b51(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:31,194 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:31,194 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/B, priority=13, startTime=1732005630706; duration=0sec 2024-11-19T08:40:31,194 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:31,195 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:B 2024-11-19T08:40:31,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:31,322 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119be47999c00634f0cb905fb9c4194d745_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be47999c00634f0cb905fb9c4194d745_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/b299b0c045a445e694ddfe37801a19c3, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:31,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/b299b0c045a445e694ddfe37801a19c3 is 175, key is test_row_0/A:col10/1732005629349/Put/seqid=0 2024-11-19T08:40:31,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741971_1147 (size=30955) 2024-11-19T08:40:31,382 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/89d3285dae62450a99c90668c6fbfa31 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/89d3285dae62450a99c90668c6fbfa31 2024-11-19T08:40:31,390 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/C of 392ed962c42d18afe3f39ea7d3a6fd3e into 89d3285dae62450a99c90668c6fbfa31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:31,390 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:31,390 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/C, priority=13, startTime=1732005630706; duration=0sec 2024-11-19T08:40:31,390 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:31,390 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:C 2024-11-19T08:40:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T08:40:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:31,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005691524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005691526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,529 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 392ed962c42d18afe3f39ea7d3a6fd3e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005691526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005691528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005691636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005691636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005691636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,729 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/b299b0c045a445e694ddfe37801a19c3 2024-11-19T08:40:31,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/fba56ac4fc6e4523a64453f95697f7a2 is 50, key is test_row_0/B:col10/1732005629349/Put/seqid=0 2024-11-19T08:40:31,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741972_1148 (size=12001) 2024-11-19T08:40:31,793 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/fba56ac4fc6e4523a64453f95697f7a2 2024-11-19T08:40:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/95703440145f40e5883df4a31a67c5ee is 50, key is test_row_0/C:col10/1732005629349/Put/seqid=0 2024-11-19T08:40:31,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741973_1149 (size=12001) 2024-11-19T08:40:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005691846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005691846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005691847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005692157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005692157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005692157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,233 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/95703440145f40e5883df4a31a67c5ee 2024-11-19T08:40:32,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/b299b0c045a445e694ddfe37801a19c3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/b299b0c045a445e694ddfe37801a19c3 2024-11-19T08:40:32,248 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/b299b0c045a445e694ddfe37801a19c3, entries=150, sequenceid=117, filesize=30.2 K 2024-11-19T08:40:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/fba56ac4fc6e4523a64453f95697f7a2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fba56ac4fc6e4523a64453f95697f7a2 2024-11-19T08:40:32,253 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fba56ac4fc6e4523a64453f95697f7a2, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T08:40:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/95703440145f40e5883df4a31a67c5ee as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95703440145f40e5883df4a31a67c5ee 2024-11-19T08:40:32,260 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95703440145f40e5883df4a31a67c5ee, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T08:40:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T08:40:32,262 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1450ms, sequenceid=117, compaction requested=false 2024-11-19T08:40:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:32,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T08:40:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T08:40:32,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T08:40:32,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-19T08:40:32,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9240 sec 2024-11-19T08:40:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T08:40:32,506 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:32,506 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T08:40:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T08:40:32,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:32,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:32,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T08:40:32,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T08:40:32,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:32,661 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T08:40:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a107960f6b4c4393a0cc4982e999a468_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005631524/Put/seqid=0 2024-11-19T08:40:32,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741974_1150 (size=12304) 2024-11-19T08:40:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:32,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a107960f6b4c4393a0cc4982e999a468_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a107960f6b4c4393a0cc4982e999a468_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0eafaf6953364508a40e1125e57c1021, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0eafaf6953364508a40e1125e57c1021 is 175, key is test_row_0/A:col10/1732005631524/Put/seqid=0 2024-11-19T08:40:32,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741975_1151 (size=31105) 2024-11-19T08:40:32,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005692720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,727 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0eafaf6953364508a40e1125e57c1021 2024-11-19T08:40:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005692721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005692724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/b5a8add9cbce413baeab259d59781ea9 is 50, key is test_row_0/B:col10/1732005631524/Put/seqid=0 2024-11-19T08:40:32,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741976_1152 (size=12151) 2024-11-19T08:40:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T08:40:32,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005692836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005692836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:32,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005692836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005693046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005693046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005693046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T08:40:33,168 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/b5a8add9cbce413baeab259d59781ea9 2024-11-19T08:40:33,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/5bcbe2fc69634751b221bba7f4b86132 is 50, key is test_row_0/C:col10/1732005631524/Put/seqid=0 2024-11-19T08:40:33,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741977_1153 (size=12151) 2024-11-19T08:40:33,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005693356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005693356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005693356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005693546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,549 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 392ed962c42d18afe3f39ea7d3a6fd3e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,579 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/5bcbe2fc69634751b221bba7f4b86132 2024-11-19T08:40:33,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0eafaf6953364508a40e1125e57c1021 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0eafaf6953364508a40e1125e57c1021 2024-11-19T08:40:33,594 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0eafaf6953364508a40e1125e57c1021, entries=150, sequenceid=134, filesize=30.4 K 2024-11-19T08:40:33,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/b5a8add9cbce413baeab259d59781ea9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b5a8add9cbce413baeab259d59781ea9 2024-11-19T08:40:33,603 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b5a8add9cbce413baeab259d59781ea9, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T08:40:33,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/5bcbe2fc69634751b221bba7f4b86132 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/5bcbe2fc69634751b221bba7f4b86132 2024-11-19T08:40:33,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/5bcbe2fc69634751b221bba7f4b86132, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T08:40:33,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 392ed962c42d18afe3f39ea7d3a6fd3e in 952ms, sequenceid=134, compaction requested=true 2024-11-19T08:40:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:33,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T08:40:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T08:40:33,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T08:40:33,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1070 sec 2024-11-19T08:40:33,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1120 sec 2024-11-19T08:40:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T08:40:33,647 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:33,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T08:40:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T08:40:33,649 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:33,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:33,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T08:40:33,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T08:40:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:33,803 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T08:40:33,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fb27473af4b47afaa598272df720cda_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005632722/Put/seqid=0 2024-11-19T08:40:33,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741978_1154 (size=12304) 2024-11-19T08:40:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:33,859 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fb27473af4b47afaa598272df720cda_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fb27473af4b47afaa598272df720cda_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:33,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/11906549749a4be6a28fee0156ca639b, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:33,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/11906549749a4be6a28fee0156ca639b is 175, key is test_row_0/A:col10/1732005632722/Put/seqid=0 2024-11-19T08:40:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741979_1155 (size=31105) 2024-11-19T08:40:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:33,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:33,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005693885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005693887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005693887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T08:40:34,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005693996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005693997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005693997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005694206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005694206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005694216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,274 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/11906549749a4be6a28fee0156ca639b 2024-11-19T08:40:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T08:40:34,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/8f8737d86a504d01b193b001850b9c5a is 50, key is test_row_0/B:col10/1732005632722/Put/seqid=0 2024-11-19T08:40:34,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741980_1156 (size=12151) 2024-11-19T08:40:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005694516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005694516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005694526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005694556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:34,562 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 392ed962c42d18afe3f39ea7d3a6fd3e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:34,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:34,689 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/8f8737d86a504d01b193b001850b9c5a 2024-11-19T08:40:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a3a27ae35b064ab3b75b0a7a2e1502df is 50, key is test_row_0/C:col10/1732005632722/Put/seqid=0 2024-11-19T08:40:34,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741981_1157 (size=12151) 2024-11-19T08:40:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T08:40:35,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005695026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:35,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:35,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005695026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005695047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:35,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a3a27ae35b064ab3b75b0a7a2e1502df 2024-11-19T08:40:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/11906549749a4be6a28fee0156ca639b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/11906549749a4be6a28fee0156ca639b 2024-11-19T08:40:35,121 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/11906549749a4be6a28fee0156ca639b, entries=150, sequenceid=156, filesize=30.4 K 2024-11-19T08:40:35,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/8f8737d86a504d01b193b001850b9c5a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/8f8737d86a504d01b193b001850b9c5a 2024-11-19T08:40:35,133 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/8f8737d86a504d01b193b001850b9c5a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T08:40:35,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a3a27ae35b064ab3b75b0a7a2e1502df as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a3a27ae35b064ab3b75b0a7a2e1502df 2024-11-19T08:40:35,153 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a3a27ae35b064ab3b75b0a7a2e1502df, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T08:40:35,155 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1353ms, sequenceid=156, compaction requested=true 2024-11-19T08:40:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T08:40:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T08:40:35,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T08:40:35,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-11-19T08:40:35,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5140 sec 2024-11-19T08:40:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T08:40:35,807 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:35,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T08:40:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T08:40:35,811 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:35,812 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:35,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T08:40:35,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T08:40:35,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:35,965 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T08:40:35,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119221a6c152b424d69b75f96ef7b784562_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005633886/Put/seqid=0 2024-11-19T08:40:35,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741982_1158 (size=12304) 2024-11-19T08:40:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:36,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005696099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005696100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005696102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T08:40:36,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005696206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005696216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005696216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:36,388 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119221a6c152b424d69b75f96ef7b784562_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119221a6c152b424d69b75f96ef7b784562_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f7173d2e39e34551972e4b383130b28e, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f7173d2e39e34551972e4b383130b28e is 175, key is test_row_0/A:col10/1732005633886/Put/seqid=0 2024-11-19T08:40:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741983_1159 (size=31105) 2024-11-19T08:40:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005696426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005696426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005696426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T08:40:36,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005696737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005696737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005696737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:36,795 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f7173d2e39e34551972e4b383130b28e 2024-11-19T08:40:36,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/34be836c5ef64d20b95a06616495fd5c is 50, key is test_row_0/B:col10/1732005633886/Put/seqid=0 2024-11-19T08:40:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741984_1160 (size=12151) 2024-11-19T08:40:36,815 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/34be836c5ef64d20b95a06616495fd5c 2024-11-19T08:40:36,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/6a3d8be535214a6296a06fb14c297e64 is 50, key is test_row_0/C:col10/1732005633886/Put/seqid=0 2024-11-19T08:40:36,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741985_1161 (size=12151) 2024-11-19T08:40:36,851 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/6a3d8be535214a6296a06fb14c297e64 2024-11-19T08:40:36,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/f7173d2e39e34551972e4b383130b28e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f7173d2e39e34551972e4b383130b28e 2024-11-19T08:40:36,868 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f7173d2e39e34551972e4b383130b28e, entries=150, sequenceid=170, filesize=30.4 K 2024-11-19T08:40:36,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/34be836c5ef64d20b95a06616495fd5c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/34be836c5ef64d20b95a06616495fd5c 2024-11-19T08:40:36,877 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/34be836c5ef64d20b95a06616495fd5c, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T08:40:36,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/6a3d8be535214a6296a06fb14c297e64 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/6a3d8be535214a6296a06fb14c297e64 2024-11-19T08:40:36,887 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/6a3d8be535214a6296a06fb14c297e64, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T08:40:36,888 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 392ed962c42d18afe3f39ea7d3a6fd3e in 923ms, sequenceid=170, compaction requested=true 2024-11-19T08:40:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T08:40:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T08:40:36,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T08:40:36,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0780 sec 2024-11-19T08:40:36,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0840 sec 2024-11-19T08:40:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T08:40:36,946 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:36,946 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T08:40:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T08:40:36,950 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:36,950 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:36,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T08:40:37,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T08:40:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:37,104 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T08:40:37,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193f5d4488e19b41ba9b2a8aa751fd1e14_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005636098/Put/seqid=0 2024-11-19T08:40:37,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741986_1162 (size=12304) 2024-11-19T08:40:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:37,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T08:40:37,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005697287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005697292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005697293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005697406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005697406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005697406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,412 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:37,565 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193f5d4488e19b41ba9b2a8aa751fd1e14_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f5d4488e19b41ba9b2a8aa751fd1e14_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:37,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8a6dffcf69574d65a2b99be4f30b497e, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:37,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8a6dffcf69574d65a2b99be4f30b497e is 175, key is test_row_0/A:col10/1732005636098/Put/seqid=0 2024-11-19T08:40:37,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741987_1163 (size=31105) 2024-11-19T08:40:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50692 deadline: 1732005697567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,575 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 392ed962c42d18afe3f39ea7d3a6fd3e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T08:40:37,579 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8a6dffcf69574d65a2b99be4f30b497e 2024-11-19T08:40:37,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/00288a2a771e43ed9ecf2ae5288bd5bb is 50, key is test_row_0/B:col10/1732005636098/Put/seqid=0 2024-11-19T08:40:37,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741988_1164 (size=12151) 2024-11-19T08:40:37,604 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/00288a2a771e43ed9ecf2ae5288bd5bb 2024-11-19T08:40:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/df58fb2e1d9f4a5a88ac643f88b28011 is 50, key is test_row_0/C:col10/1732005636098/Put/seqid=0 2024-11-19T08:40:37,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741989_1165 (size=12151) 2024-11-19T08:40:37,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005697616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005697616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005697616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,837 INFO [master/3ab37fa97a98:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T08:40:37,837 INFO [master/3ab37fa97a98:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T08:40:37,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005697935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005697936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005697936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:37,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,022 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/df58fb2e1d9f4a5a88ac643f88b28011 2024-11-19T08:40:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8a6dffcf69574d65a2b99be4f30b497e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8a6dffcf69574d65a2b99be4f30b497e 2024-11-19T08:40:38,036 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8a6dffcf69574d65a2b99be4f30b497e, entries=150, sequenceid=192, filesize=30.4 K 2024-11-19T08:40:38,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/00288a2a771e43ed9ecf2ae5288bd5bb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/00288a2a771e43ed9ecf2ae5288bd5bb 2024-11-19T08:40:38,043 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/00288a2a771e43ed9ecf2ae5288bd5bb, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T08:40:38,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/df58fb2e1d9f4a5a88ac643f88b28011 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/df58fb2e1d9f4a5a88ac643f88b28011 2024-11-19T08:40:38,052 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/df58fb2e1d9f4a5a88ac643f88b28011, entries=150, sequenceid=192, filesize=11.9 K 2024-11-19T08:40:38,053 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 392ed962c42d18afe3f39ea7d3a6fd3e in 948ms, sequenceid=192, compaction requested=true 2024-11-19T08:40:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T08:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T08:40:38,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T08:40:38,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1030 sec 2024-11-19T08:40:38,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.1110 sec 2024-11-19T08:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T08:40:38,087 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:38,088 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T08:40:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T08:40:38,090 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:38,091 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:38,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T08:40:38,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T08:40:38,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:38,244 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T08:40:38,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e13d3f28d11e4c2db781c339dac234ee_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005637283/Put/seqid=0 2024-11-19T08:40:38,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741990_1166 (size=12304) 2024-11-19T08:40:38,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:38,267 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e13d3f28d11e4c2db781c339dac234ee_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e13d3f28d11e4c2db781c339dac234ee_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/44562cffe8f6468c9bc9f0983c39f9d9, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/44562cffe8f6468c9bc9f0983c39f9d9 is 175, key is test_row_0/A:col10/1732005637283/Put/seqid=0 2024-11-19T08:40:38,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741991_1167 (size=31105) 2024-11-19T08:40:38,273 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/44562cffe8f6468c9bc9f0983c39f9d9 2024-11-19T08:40:38,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/5d755d3bf77b4b7eb8dabf7da6749dbf is 50, key is test_row_0/B:col10/1732005637283/Put/seqid=0 2024-11-19T08:40:38,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741992_1168 (size=12151) 2024-11-19T08:40:38,285 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/5d755d3bf77b4b7eb8dabf7da6749dbf 2024-11-19T08:40:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/058db340d88d449b91892ef7f9658c3e is 50, key is test_row_0/C:col10/1732005637283/Put/seqid=0 2024-11-19T08:40:38,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741993_1169 (size=12151) 2024-11-19T08:40:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T08:40:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:38,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005698479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005698480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005698490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005698595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005698596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005698596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,708 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/058db340d88d449b91892ef7f9658c3e 2024-11-19T08:40:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/44562cffe8f6468c9bc9f0983c39f9d9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/44562cffe8f6468c9bc9f0983c39f9d9 2024-11-19T08:40:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T08:40:38,719 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/44562cffe8f6468c9bc9f0983c39f9d9, entries=150, sequenceid=206, filesize=30.4 K 2024-11-19T08:40:38,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/5d755d3bf77b4b7eb8dabf7da6749dbf as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/5d755d3bf77b4b7eb8dabf7da6749dbf 2024-11-19T08:40:38,726 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/5d755d3bf77b4b7eb8dabf7da6749dbf, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T08:40:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/058db340d88d449b91892ef7f9658c3e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/058db340d88d449b91892ef7f9658c3e 2024-11-19T08:40:38,732 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/058db340d88d449b91892ef7f9658c3e, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T08:40:38,733 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 392ed962c42d18afe3f39ea7d3a6fd3e in 489ms, sequenceid=206, compaction requested=true 2024-11-19T08:40:38,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:38,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:38,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T08:40:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T08:40:38,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T08:40:38,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 644 msec 2024-11-19T08:40:38,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 651 msec 2024-11-19T08:40:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:38,811 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:38,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T08:40:38,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005698832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005698832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005698833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194f2fcb33943b409c8b2b6f2f79877557_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005638488/Put/seqid=0 2024-11-19T08:40:38,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741994_1170 (size=17284) 2024-11-19T08:40:38,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:38,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194f2fcb33943b409c8b2b6f2f79877557_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194f2fcb33943b409c8b2b6f2f79877557_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:38,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/77a7376ed1034e3cad8244e3d0ed6e2e, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:38,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/77a7376ed1034e3cad8244e3d0ed6e2e is 175, key is test_row_0/A:col10/1732005638488/Put/seqid=0 2024-11-19T08:40:38,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741995_1171 (size=48389) 2024-11-19T08:40:38,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005698946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005698946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005698946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005699155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005699156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005699156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T08:40:39,226 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:39,226 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-19T08:40:39,228 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T08:40:39,229 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:39,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:39,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/77a7376ed1034e3cad8244e3d0ed6e2e 2024-11-19T08:40:39,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/a92b2c84796d4ace9f304e37147ea761 is 50, key is test_row_0/B:col10/1732005638488/Put/seqid=0 2024-11-19T08:40:39,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741996_1172 (size=12151) 2024-11-19T08:40:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T08:40:39,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T08:40:39,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:39,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005699465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005699465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005699466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T08:40:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T08:40:39,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T08:40:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/a92b2c84796d4ace9f304e37147ea761 2024-11-19T08:40:39,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/4572d8710db04992a7dd53b4468c5d18 is 50, key is test_row_0/C:col10/1732005638488/Put/seqid=0 2024-11-19T08:40:39,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741997_1173 (size=12151) 2024-11-19T08:40:39,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T08:40:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T08:40:39,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005699986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005699986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005699986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:39,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:39,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:39,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T08:40:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:39,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:40,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/4572d8710db04992a7dd53b4468c5d18 2024-11-19T08:40:40,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/77a7376ed1034e3cad8244e3d0ed6e2e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/77a7376ed1034e3cad8244e3d0ed6e2e 2024-11-19T08:40:40,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/77a7376ed1034e3cad8244e3d0ed6e2e, entries=250, sequenceid=230, filesize=47.3 K 2024-11-19T08:40:40,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/a92b2c84796d4ace9f304e37147ea761 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/a92b2c84796d4ace9f304e37147ea761 2024-11-19T08:40:40,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/a92b2c84796d4ace9f304e37147ea761, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T08:40:40,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/4572d8710db04992a7dd53b4468c5d18 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/4572d8710db04992a7dd53b4468c5d18 2024-11-19T08:40:40,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/4572d8710db04992a7dd53b4468c5d18, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T08:40:40,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1324ms, sequenceid=230, compaction requested=true 2024-11-19T08:40:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:40,136 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-19T08:40:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:40,136 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-19T08:40:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:40,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:40,140 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 97114 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-19T08:40:40,140 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/B is initiating minor compaction (all files) 2024-11-19T08:40:40,140 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/B in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:40,140 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/634e7f7ecfad4e64bcd8588636184b51, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fba56ac4fc6e4523a64453f95697f7a2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b5a8add9cbce413baeab259d59781ea9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/8f8737d86a504d01b193b001850b9c5a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/34be836c5ef64d20b95a06616495fd5c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/00288a2a771e43ed9ecf2ae5288bd5bb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/5d755d3bf77b4b7eb8dabf7da6749dbf, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/a92b2c84796d4ace9f304e37147ea761] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=94.8 K 2024-11-19T08:40:40,141 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 266030 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-19T08:40:40,141 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/A is initiating minor compaction (all files) 2024-11-19T08:40:40,141 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/A in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:40,141 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/9a67e94053c4431ebf0b20cd510bfed5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/b299b0c045a445e694ddfe37801a19c3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0eafaf6953364508a40e1125e57c1021, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/11906549749a4be6a28fee0156ca639b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f7173d2e39e34551972e4b383130b28e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8a6dffcf69574d65a2b99be4f30b497e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/44562cffe8f6468c9bc9f0983c39f9d9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/77a7376ed1034e3cad8244e3d0ed6e2e] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=259.8 K 2024-11-19T08:40:40,141 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=8 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:40,142 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/9a67e94053c4431ebf0b20cd510bfed5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/b299b0c045a445e694ddfe37801a19c3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0eafaf6953364508a40e1125e57c1021, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/11906549749a4be6a28fee0156ca639b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f7173d2e39e34551972e4b383130b28e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8a6dffcf69574d65a2b99be4f30b497e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/44562cffe8f6468c9bc9f0983c39f9d9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/77a7376ed1034e3cad8244e3d0ed6e2e] 2024-11-19T08:40:40,142 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 634e7f7ecfad4e64bcd8588636184b51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005628110 2024-11-19T08:40:40,142 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a67e94053c4431ebf0b20cd510bfed5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005628110 2024-11-19T08:40:40,143 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting fba56ac4fc6e4523a64453f95697f7a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732005629341 2024-11-19T08:40:40,143 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting b299b0c045a445e694ddfe37801a19c3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732005629341 2024-11-19T08:40:40,143 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a8add9cbce413baeab259d59781ea9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005631521 2024-11-19T08:40:40,144 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8737d86a504d01b193b001850b9c5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732005632715 2024-11-19T08:40:40,144 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eafaf6953364508a40e1125e57c1021, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005631521 2024-11-19T08:40:40,145 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11906549749a4be6a28fee0156ca639b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732005632715 2024-11-19T08:40:40,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T08:40:40,145 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 34be836c5ef64d20b95a06616495fd5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732005633876 2024-11-19T08:40:40,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:40,146 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:40:40,147 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 00288a2a771e43ed9ecf2ae5288bd5bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732005636089 2024-11-19T08:40:40,147 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7173d2e39e34551972e4b383130b28e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732005633876 2024-11-19T08:40:40,147 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a6dffcf69574d65a2b99be4f30b497e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732005636089 2024-11-19T08:40:40,147 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d755d3bf77b4b7eb8dabf7da6749dbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732005637273 2024-11-19T08:40:40,148 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a92b2c84796d4ace9f304e37147ea761, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005638488 2024-11-19T08:40:40,148 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44562cffe8f6468c9bc9f0983c39f9d9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732005637273 2024-11-19T08:40:40,149 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a7376ed1034e3cad8244e3d0ed6e2e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005638473 2024-11-19T08:40:40,175 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:40,189 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#B#compaction#145 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:40,190 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/742b37cc3d76449299213c2083771450 is 50, key is test_row_0/B:col10/1732005638488/Put/seqid=0 2024-11-19T08:40:40,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196bae8b38deac44c18d7e4bdf865949d4_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005638825/Put/seqid=0 2024-11-19T08:40:40,199 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199ecfb88ef02347d9ae419e7357055755_392ed962c42d18afe3f39ea7d3a6fd3e store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:40,205 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199ecfb88ef02347d9ae419e7357055755_392ed962c42d18afe3f39ea7d3a6fd3e, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:40,205 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199ecfb88ef02347d9ae419e7357055755_392ed962c42d18afe3f39ea7d3a6fd3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:40,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741999_1175 (size=12629) 2024-11-19T08:40:40,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741998_1174 (size=12304) 2024-11-19T08:40:40,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742000_1176 (size=4469) 2024-11-19T08:40:40,231 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#A#compaction#144 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:40,232 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/e49bc10ec40f4696852cf24e1e1c3d1c is 175, key is test_row_0/A:col10/1732005638488/Put/seqid=0 2024-11-19T08:40:40,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742001_1177 (size=31583) 2024-11-19T08:40:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T08:40:40,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:40,630 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/742b37cc3d76449299213c2083771450 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/742b37cc3d76449299213c2083771450 2024-11-19T08:40:40,632 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196bae8b38deac44c18d7e4bdf865949d4_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196bae8b38deac44c18d7e4bdf865949d4_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/406923731efe4551a631516246703b04, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:40,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/406923731efe4551a631516246703b04 is 175, key is test_row_0/A:col10/1732005638825/Put/seqid=0 2024-11-19T08:40:40,640 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/B of 392ed962c42d18afe3f39ea7d3a6fd3e into 742b37cc3d76449299213c2083771450(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:40,640 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:40,640 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/B, priority=8, startTime=1732005640136; duration=0sec 2024-11-19T08:40:40,640 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:40,640 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:B 2024-11-19T08:40:40,640 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-19T08:40:40,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742002_1178 (size=31105) 2024-11-19T08:40:40,644 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/e49bc10ec40f4696852cf24e1e1c3d1c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/e49bc10ec40f4696852cf24e1e1c3d1c 2024-11-19T08:40:40,646 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 97114 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-19T08:40:40,646 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/C is initiating minor compaction (all files) 2024-11-19T08:40:40,646 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/C in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:40,646 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/89d3285dae62450a99c90668c6fbfa31, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95703440145f40e5883df4a31a67c5ee, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/5bcbe2fc69634751b221bba7f4b86132, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a3a27ae35b064ab3b75b0a7a2e1502df, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/6a3d8be535214a6296a06fb14c297e64, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/df58fb2e1d9f4a5a88ac643f88b28011, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/058db340d88d449b91892ef7f9658c3e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/4572d8710db04992a7dd53b4468c5d18] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=94.8 K 2024-11-19T08:40:40,652 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d3285dae62450a99c90668c6fbfa31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005628110 2024-11-19T08:40:40,653 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 95703440145f40e5883df4a31a67c5ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732005629341 2024-11-19T08:40:40,653 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bcbe2fc69634751b221bba7f4b86132, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005631521 2024-11-19T08:40:40,654 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a3a27ae35b064ab3b75b0a7a2e1502df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732005632715 2024-11-19T08:40:40,655 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3d8be535214a6296a06fb14c297e64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732005633876 2024-11-19T08:40:40,656 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/A of 392ed962c42d18afe3f39ea7d3a6fd3e into e49bc10ec40f4696852cf24e1e1c3d1c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:40,656 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:40,656 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/A, priority=8, startTime=1732005640136; duration=0sec 2024-11-19T08:40:40,656 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:40,656 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:A 2024-11-19T08:40:40,657 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting df58fb2e1d9f4a5a88ac643f88b28011, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732005636089 2024-11-19T08:40:40,658 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 058db340d88d449b91892ef7f9658c3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732005637273 2024-11-19T08:40:40,658 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4572d8710db04992a7dd53b4468c5d18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005638488 2024-11-19T08:40:40,671 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#C#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:40,672 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/0d6afc006c4247c88296d7435ea80b5c is 50, key is test_row_0/C:col10/1732005638488/Put/seqid=0 2024-11-19T08:40:40,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742003_1179 (size=12629) 2024-11-19T08:40:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:40,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:41,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005701032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005701035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,044 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/406923731efe4551a631516246703b04 2024-11-19T08:40:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005701037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/2bc8bd2d35a14bb88325fefd51351545 is 50, key is test_row_0/B:col10/1732005638825/Put/seqid=0 2024-11-19T08:40:41,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742004_1180 (size=12151) 2024-11-19T08:40:41,080 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/0d6afc006c4247c88296d7435ea80b5c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0d6afc006c4247c88296d7435ea80b5c 2024-11-19T08:40:41,087 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/C of 392ed962c42d18afe3f39ea7d3a6fd3e into 0d6afc006c4247c88296d7435ea80b5c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:41,087 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:41,087 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/C, priority=8, startTime=1732005640136; duration=0sec 2024-11-19T08:40:41,088 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:41,088 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:C 2024-11-19T08:40:41,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005701145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005701156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005701156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005701355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005701366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005701366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T08:40:41,457 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/2bc8bd2d35a14bb88325fefd51351545 2024-11-19T08:40:41,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a0c4bfb5cf9345feb85963cdc1c56298 is 50, key is test_row_0/C:col10/1732005638825/Put/seqid=0 2024-11-19T08:40:41,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742005_1181 (size=12151) 2024-11-19T08:40:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005701666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005701676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005701676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:41,872 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a0c4bfb5cf9345feb85963cdc1c56298 2024-11-19T08:40:41,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/406923731efe4551a631516246703b04 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/406923731efe4551a631516246703b04 2024-11-19T08:40:41,895 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/406923731efe4551a631516246703b04, entries=150, sequenceid=242, filesize=30.4 K 2024-11-19T08:40:41,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/2bc8bd2d35a14bb88325fefd51351545 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2bc8bd2d35a14bb88325fefd51351545 2024-11-19T08:40:41,902 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2bc8bd2d35a14bb88325fefd51351545, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T08:40:41,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/a0c4bfb5cf9345feb85963cdc1c56298 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a0c4bfb5cf9345feb85963cdc1c56298 2024-11-19T08:40:41,918 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a0c4bfb5cf9345feb85963cdc1c56298, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T08:40:41,919 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1773ms, sequenceid=242, compaction requested=false 2024-11-19T08:40:41,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:41,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T08:40:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T08:40:41,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T08:40:41,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6930 sec 2024-11-19T08:40:41,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7020 sec 2024-11-19T08:40:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:42,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:42,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:40:42,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119170e926c30f3441a9dec75ca10ec1b5a_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005642177/Put/seqid=0 2024-11-19T08:40:42,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742006_1182 (size=14994) 2024-11-19T08:40:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005702232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005702242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005702242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005702346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005702356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005702357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005702556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005702566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005702566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,615 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:42,628 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119170e926c30f3441a9dec75ca10ec1b5a_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119170e926c30f3441a9dec75ca10ec1b5a_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:42,631 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/c10d14f266b341399a0fc13b6df9a61a, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:42,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/c10d14f266b341399a0fc13b6df9a61a is 175, key is test_row_0/A:col10/1732005642177/Put/seqid=0 2024-11-19T08:40:42,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742007_1183 (size=39949) 2024-11-19T08:40:42,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/c10d14f266b341399a0fc13b6df9a61a 2024-11-19T08:40:42,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/75674d4d45d0499a97da09d076b066a3 is 50, key is test_row_0/B:col10/1732005642177/Put/seqid=0 2024-11-19T08:40:42,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742008_1184 (size=12301) 2024-11-19T08:40:42,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/75674d4d45d0499a97da09d076b066a3 2024-11-19T08:40:42,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/d703b5377ab04b80ab92ab76a2cdf079 is 50, key is test_row_0/C:col10/1732005642177/Put/seqid=0 2024-11-19T08:40:42,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742009_1185 (size=12301) 2024-11-19T08:40:42,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005702866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005702877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005702877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:42,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/d703b5377ab04b80ab92ab76a2cdf079 2024-11-19T08:40:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/c10d14f266b341399a0fc13b6df9a61a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c10d14f266b341399a0fc13b6df9a61a 2024-11-19T08:40:43,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c10d14f266b341399a0fc13b6df9a61a, entries=200, sequenceid=270, filesize=39.0 K 2024-11-19T08:40:43,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/75674d4d45d0499a97da09d076b066a3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/75674d4d45d0499a97da09d076b066a3 2024-11-19T08:40:43,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/75674d4d45d0499a97da09d076b066a3, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T08:40:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/d703b5377ab04b80ab92ab76a2cdf079 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d703b5377ab04b80ab92ab76a2cdf079 2024-11-19T08:40:43,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d703b5377ab04b80ab92ab76a2cdf079, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T08:40:43,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 392ed962c42d18afe3f39ea7d3a6fd3e in 976ms, sequenceid=270, compaction requested=true 2024-11-19T08:40:43,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:43,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:43,155 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:43,155 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:43,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:43,156 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:43,156 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:43,156 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/A is initiating minor compaction (all files) 2024-11-19T08:40:43,156 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/A in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,156 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/e49bc10ec40f4696852cf24e1e1c3d1c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/406923731efe4551a631516246703b04, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c10d14f266b341399a0fc13b6df9a61a] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=100.2 K 2024-11-19T08:40:43,156 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,156 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/e49bc10ec40f4696852cf24e1e1c3d1c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/406923731efe4551a631516246703b04, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c10d14f266b341399a0fc13b6df9a61a] 2024-11-19T08:40:43,159 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/B is initiating minor compaction (all files) 2024-11-19T08:40:43,159 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/B in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,159 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/742b37cc3d76449299213c2083771450, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2bc8bd2d35a14bb88325fefd51351545, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/75674d4d45d0499a97da09d076b066a3] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=36.2 K 2024-11-19T08:40:43,160 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting e49bc10ec40f4696852cf24e1e1c3d1c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005638488 2024-11-19T08:40:43,160 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 742b37cc3d76449299213c2083771450, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005638488 2024-11-19T08:40:43,160 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 406923731efe4551a631516246703b04, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732005638825 2024-11-19T08:40:43,161 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10d14f266b341399a0fc13b6df9a61a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732005641022 2024-11-19T08:40:43,161 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc8bd2d35a14bb88325fefd51351545, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732005638825 2024-11-19T08:40:43,161 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 75674d4d45d0499a97da09d076b066a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732005641022 2024-11-19T08:40:43,177 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:43,179 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:43,179 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/2623ce4b9c57468eb87d868e0373d1b2 is 50, key is test_row_0/B:col10/1732005642177/Put/seqid=0 2024-11-19T08:40:43,192 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111950f2e297f1f54378b25eb24f17334e80_392ed962c42d18afe3f39ea7d3a6fd3e store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:43,194 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111950f2e297f1f54378b25eb24f17334e80_392ed962c42d18afe3f39ea7d3a6fd3e, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:43,195 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111950f2e297f1f54378b25eb24f17334e80_392ed962c42d18afe3f39ea7d3a6fd3e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742011_1187 (size=4469) 2024-11-19T08:40:43,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742010_1186 (size=12881) 2024-11-19T08:40:43,251 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/2623ce4b9c57468eb87d868e0373d1b2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2623ce4b9c57468eb87d868e0373d1b2 2024-11-19T08:40:43,267 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/B of 392ed962c42d18afe3f39ea7d3a6fd3e into 2623ce4b9c57468eb87d868e0373d1b2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:43,267 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:43,267 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/B, priority=13, startTime=1732005643155; duration=0sec 2024-11-19T08:40:43,267 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:43,267 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:B 2024-11-19T08:40:43,268 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:40:43,270 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:40:43,270 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 392ed962c42d18afe3f39ea7d3a6fd3e/C is initiating minor compaction (all files) 2024-11-19T08:40:43,271 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 392ed962c42d18afe3f39ea7d3a6fd3e/C in TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,271 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0d6afc006c4247c88296d7435ea80b5c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a0c4bfb5cf9345feb85963cdc1c56298, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d703b5377ab04b80ab92ab76a2cdf079] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp, totalSize=36.2 K 2024-11-19T08:40:43,271 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d6afc006c4247c88296d7435ea80b5c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005638488 2024-11-19T08:40:43,272 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a0c4bfb5cf9345feb85963cdc1c56298, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732005638825 2024-11-19T08:40:43,272 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d703b5377ab04b80ab92ab76a2cdf079, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732005641022 2024-11-19T08:40:43,305 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:43,306 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/960b037e4375426c943083b2d254ea6e is 50, key is test_row_0/C:col10/1732005642177/Put/seqid=0 2024-11-19T08:40:43,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742012_1188 (size=12881) 2024-11-19T08:40:43,351 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/960b037e4375426c943083b2d254ea6e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/960b037e4375426c943083b2d254ea6e 2024-11-19T08:40:43,361 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/C of 392ed962c42d18afe3f39ea7d3a6fd3e into 960b037e4375426c943083b2d254ea6e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:43,361 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:43,361 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/C, priority=13, startTime=1732005643155; duration=0sec 2024-11-19T08:40:43,361 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:43,361 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:C 2024-11-19T08:40:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:43,377 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:43,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T08:40:43,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fc5f0426560f4d77b66841647d1eb7fa_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005642240/Put/seqid=0 2024-11-19T08:40:43,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742013_1189 (size=12454) 2024-11-19T08:40:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T08:40:43,406 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:43,406 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:43,407 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-19T08:40:43,412 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:43,412 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fc5f0426560f4d77b66841647d1eb7fa_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fc5f0426560f4d77b66841647d1eb7fa_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T08:40:43,414 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/140a0a9b99bf4699a09e4840492dce93, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:43,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/140a0a9b99bf4699a09e4840492dce93 is 175, key is test_row_0/A:col10/1732005642240/Put/seqid=0 2024-11-19T08:40:43,414 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:43,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:43,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005703427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742014_1190 (size=31255) 2024-11-19T08:40:43,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005703433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005703433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T08:40:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005703536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005703546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005703546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T08:40:43,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:43,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,638 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 392ed962c42d18afe3f39ea7d3a6fd3e#A#compaction#154 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:43,638 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8c2e97f2af8e42a895734760ba0bee43 is 175, key is test_row_0/A:col10/1732005642177/Put/seqid=0 2024-11-19T08:40:43,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742015_1191 (size=31835) 2024-11-19T08:40:43,653 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/8c2e97f2af8e42a895734760ba0bee43 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8c2e97f2af8e42a895734760ba0bee43 2024-11-19T08:40:43,660 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 392ed962c42d18afe3f39ea7d3a6fd3e/A of 392ed962c42d18afe3f39ea7d3a6fd3e into 8c2e97f2af8e42a895734760ba0bee43(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:43,660 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:43,660 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., storeName=392ed962c42d18afe3f39ea7d3a6fd3e/A, priority=13, startTime=1732005643154; duration=0sec 2024-11-19T08:40:43,660 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:43,660 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:A 2024-11-19T08:40:43,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T08:40:43,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:43,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T08:40:43,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005703746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005703756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005703756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:43,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/140a0a9b99bf4699a09e4840492dce93 2024-11-19T08:40:43,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/30665d22a00d41dfa77aa38a275704a7 is 50, key is test_row_0/B:col10/1732005642240/Put/seqid=0 2024-11-19T08:40:43,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742016_1192 (size=12301) 2024-11-19T08:40:43,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T08:40:43,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:43,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:43,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T08:40:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:44,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T08:40:44,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005704056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005704066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005704066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T08:40:44,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:44,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:44,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:44,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/30665d22a00d41dfa77aa38a275704a7 2024-11-19T08:40:44,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/e269878ce427474db3301be50023023b is 50, key is test_row_0/C:col10/1732005642240/Put/seqid=0 2024-11-19T08:40:44,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742017_1193 (size=12301) 2024-11-19T08:40:44,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/e269878ce427474db3301be50023023b 2024-11-19T08:40:44,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/140a0a9b99bf4699a09e4840492dce93 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/140a0a9b99bf4699a09e4840492dce93 2024-11-19T08:40:44,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/140a0a9b99bf4699a09e4840492dce93, entries=150, sequenceid=285, filesize=30.5 K 2024-11-19T08:40:44,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/30665d22a00d41dfa77aa38a275704a7 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/30665d22a00d41dfa77aa38a275704a7 2024-11-19T08:40:44,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/30665d22a00d41dfa77aa38a275704a7, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T08:40:44,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/e269878ce427474db3301be50023023b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/e269878ce427474db3301be50023023b 2024-11-19T08:40:44,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T08:40:44,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:44,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:44,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:44,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:44,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/e269878ce427474db3301be50023023b, entries=150, sequenceid=285, filesize=12.0 K 2024-11-19T08:40:44,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 392ed962c42d18afe3f39ea7d3a6fd3e in 971ms, sequenceid=285, compaction requested=false 2024-11-19T08:40:44,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:44,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T08:40:44,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:44,494 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:40:44,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a875882617b74520af9afde450d2767d_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005643432/Put/seqid=0 2024-11-19T08:40:44,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742018_1194 (size=12454) 2024-11-19T08:40:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T08:40:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005704598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005704599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005704601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005704602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005704706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005704706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005704706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005704706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005704926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005704926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005704926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005704927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:44,948 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a875882617b74520af9afde450d2767d_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a875882617b74520af9afde450d2767d_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:44,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ed9b4c375b754a138384121b56224666, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:44,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ed9b4c375b754a138384121b56224666 is 175, key is test_row_0/A:col10/1732005643432/Put/seqid=0 2024-11-19T08:40:44,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742019_1195 (size=31255) 2024-11-19T08:40:44,975 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ed9b4c375b754a138384121b56224666 2024-11-19T08:40:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/82f6127237e742e5b32f356d179bd04b is 50, key is test_row_0/B:col10/1732005643432/Put/seqid=0 2024-11-19T08:40:44,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742020_1196 (size=12301) 2024-11-19T08:40:45,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005705236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005705236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005705246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005705246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,394 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/82f6127237e742e5b32f356d179bd04b 2024-11-19T08:40:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/1867bd00dd6345cdad7e4f7f80191585 is 50, key is test_row_0/C:col10/1732005643432/Put/seqid=0 2024-11-19T08:40:45,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742021_1197 (size=12301) 2024-11-19T08:40:45,422 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/1867bd00dd6345cdad7e4f7f80191585 2024-11-19T08:40:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ed9b4c375b754a138384121b56224666 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ed9b4c375b754a138384121b56224666 2024-11-19T08:40:45,441 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ed9b4c375b754a138384121b56224666, entries=150, sequenceid=309, filesize=30.5 K 2024-11-19T08:40:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/82f6127237e742e5b32f356d179bd04b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/82f6127237e742e5b32f356d179bd04b 2024-11-19T08:40:45,450 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/82f6127237e742e5b32f356d179bd04b, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T08:40:45,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/1867bd00dd6345cdad7e4f7f80191585 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/1867bd00dd6345cdad7e4f7f80191585 2024-11-19T08:40:45,459 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/1867bd00dd6345cdad7e4f7f80191585, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T08:40:45,460 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 392ed962c42d18afe3f39ea7d3a6fd3e in 966ms, sequenceid=309, compaction requested=true 2024-11-19T08:40:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:45,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-19T08:40:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-19T08:40:45,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T08:40:45,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0480 sec 2024-11-19T08:40:45,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.0580 sec 2024-11-19T08:40:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T08:40:45,566 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:45,566 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-19T08:40:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T08:40:45,569 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:45,569 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:45,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T08:40:45,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T08:40:45,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:45,723 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T08:40:45,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f9fca11202604854a1e8150f4a2f17be_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005644597/Put/seqid=0 2024-11-19T08:40:45,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742022_1198 (size=12454) 2024-11-19T08:40:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:45,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. as already flushing 2024-11-19T08:40:45,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005705777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005705779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005705779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005705782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T08:40:45,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005705886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005705886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005705886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:45,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005705887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005706096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005706096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005706096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005706096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:46,174 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-11-19T08:40:46,174 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:46,174 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,174 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:46,174 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-19T08:40:46,175 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:46,175 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,175 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:46,177 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-11-19T08:40:46,177 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:46,177 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,177 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:46,180 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-11-19T08:40:46,180 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:46,180 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,180 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f9fca11202604854a1e8150f4a2f17be_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f9fca11202604854a1e8150f4a2f17be_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:46,180 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:46,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:46,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ab956f3fd42d4216ba82fa86f6ff8aab, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:46,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ab956f3fd42d4216ba82fa86f6ff8aab is 175, key is test_row_0/A:col10/1732005644597/Put/seqid=0 2024-11-19T08:40:46,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742023_1199 (size=31255) 2024-11-19T08:40:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T08:40:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005706407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005706407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005706407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005706407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ab956f3fd42d4216ba82fa86f6ff8aab 2024-11-19T08:40:46,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/efbf576018d34ec3a6f27b21a9b25f07 is 50, key is test_row_0/B:col10/1732005644597/Put/seqid=0 2024-11-19T08:40:46,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742024_1200 (size=12301) 2024-11-19T08:40:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T08:40:46,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50690 deadline: 1732005706916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50710 deadline: 1732005706916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50708 deadline: 1732005706917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50720 deadline: 1732005706917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:40:47,011 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/efbf576018d34ec3a6f27b21a9b25f07 2024-11-19T08:40:47,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/0fb8069a39574296994af8468286da32 is 50, key is test_row_0/C:col10/1732005644597/Put/seqid=0 2024-11-19T08:40:47,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742025_1201 (size=12301) 2024-11-19T08:40:47,429 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/0fb8069a39574296994af8468286da32 2024-11-19T08:40:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/ab956f3fd42d4216ba82fa86f6ff8aab as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ab956f3fd42d4216ba82fa86f6ff8aab 2024-11-19T08:40:47,444 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ab956f3fd42d4216ba82fa86f6ff8aab, entries=150, sequenceid=322, filesize=30.5 K 2024-11-19T08:40:47,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/efbf576018d34ec3a6f27b21a9b25f07 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/efbf576018d34ec3a6f27b21a9b25f07 2024-11-19T08:40:47,451 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/efbf576018d34ec3a6f27b21a9b25f07, entries=150, sequenceid=322, filesize=12.0 K 2024-11-19T08:40:47,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/0fb8069a39574296994af8468286da32 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0fb8069a39574296994af8468286da32 2024-11-19T08:40:47,456 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0fb8069a39574296994af8468286da32, entries=150, sequenceid=322, filesize=12.0 K 2024-11-19T08:40:47,457 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1734ms, sequenceid=322, compaction requested=true 2024-11-19T08:40:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-19T08:40:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-19T08:40:47,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T08:40:47,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8880 sec 2024-11-19T08:40:47,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.8950 sec 2024-11-19T08:40:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:47,608 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:47,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T08:40:47,609 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-19T08:40:47,609 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:47,609 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,609 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:47,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192e36dc8530c04aac9ba56a16764fee18_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005645777/Put/seqid=0 2024-11-19T08:40:47,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742026_1202 (size=12454) 2024-11-19T08:40:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T08:40:47,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:47,929 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-19T08:40:47,929 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-19T08:40:47,929 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:47,929 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:47,929 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,929 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,929 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:47,929 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:47,938 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-19T08:40:47,938 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-19T08:40:47,938 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:47,938 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,938 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:40:47,938 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,938 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:47,938 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2879 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2864 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1304 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3912 rows 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1301 2024-11-19T08:40:47,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3897 rows 2024-11-19T08:40:47,939 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:40:47,939 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:40:47,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:47,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:40:47,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T08:40:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T08:40:47,945 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005647945"}]},"ts":"1732005647945"} 2024-11-19T08:40:47,947 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T08:40:47,947 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T08:40:47,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:40:47,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, UNASSIGN}] 2024-11-19T08:40:47,950 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, UNASSIGN 2024-11-19T08:40:47,950 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:47,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, UNASSIGN because future has completed 2024-11-19T08:40:47,952 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:40:47,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:40:48,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:48,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192e36dc8530c04aac9ba56a16764fee18_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192e36dc8530c04aac9ba56a16764fee18_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:48,027 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/3ec60af824534d5680ab6d951e16f512, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:48,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/3ec60af824534d5680ab6d951e16f512 is 175, key is test_row_0/A:col10/1732005645777/Put/seqid=0 2024-11-19T08:40:48,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742027_1203 (size=31255) 2024-11-19T08:40:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T08:40:48,104 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:48,104 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:40:48,104 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 392ed962c42d18afe3f39ea7d3a6fd3e, disabling compactions & flushes 2024-11-19T08:40:48,104 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T08:40:48,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/3ec60af824534d5680ab6d951e16f512 2024-11-19T08:40:48,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/6d853d2184ff469a9b164c1edf2de233 is 50, key is test_row_0/B:col10/1732005645777/Put/seqid=0 2024-11-19T08:40:48,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742028_1204 (size=12301) 2024-11-19T08:40:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T08:40:48,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/6d853d2184ff469a9b164c1edf2de233 2024-11-19T08:40:48,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/95d5c351908c4bf894a42ad99b20428f is 50, key is test_row_0/C:col10/1732005645777/Put/seqid=0 2024-11-19T08:40:48,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742029_1205 (size=12301) 2024-11-19T08:40:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T08:40:49,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/95d5c351908c4bf894a42ad99b20428f 2024-11-19T08:40:49,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/3ec60af824534d5680ab6d951e16f512 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/3ec60af824534d5680ab6d951e16f512 2024-11-19T08:40:49,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/3ec60af824534d5680ab6d951e16f512, entries=150, sequenceid=346, filesize=30.5 K 2024-11-19T08:40:49,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/6d853d2184ff469a9b164c1edf2de233 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6d853d2184ff469a9b164c1edf2de233 2024-11-19T08:40:49,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6d853d2184ff469a9b164c1edf2de233, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T08:40:49,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/95d5c351908c4bf894a42ad99b20428f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95d5c351908c4bf894a42ad99b20428f 2024-11-19T08:40:49,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95d5c351908c4bf894a42ad99b20428f, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T08:40:49,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1684ms, sequenceid=346, compaction requested=true 2024-11-19T08:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 392ed962c42d18afe3f39ea7d3a6fd3e: 2024-11-19T08:40:49,292 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:49,292 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:49,292 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. because compaction request was cancelled 2024-11-19T08:40:49,292 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. after waiting 0 ms 2024-11-19T08:40:49,292 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:49,292 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:A 2024-11-19T08:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:B, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:49,292 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. because compaction request was cancelled 2024-11-19T08:40:49,292 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 392ed962c42d18afe3f39ea7d3a6fd3e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T08:40:49,292 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:B 2024-11-19T08:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 392ed962c42d18afe3f39ea7d3a6fd3e:C, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:49,292 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. because compaction request was cancelled 2024-11-19T08:40:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:49,292 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 392ed962c42d18afe3f39ea7d3a6fd3e:C 2024-11-19T08:40:49,297 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f86a7ab002e0470b8d64bb385d273273_392ed962c42d18afe3f39ea7d3a6fd3e is 50, key is test_row_0/A:col10/1732005647936/Put/seqid=0 2024-11-19T08:40:49,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742030_1206 (size=12454) 2024-11-19T08:40:49,701 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:49,706 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f86a7ab002e0470b8d64bb385d273273_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f86a7ab002e0470b8d64bb385d273273_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:49,707 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0e1ad619ceb54e96836beb6087ad8bb1, store: [table=TestAcidGuarantees family=A region=392ed962c42d18afe3f39ea7d3a6fd3e] 2024-11-19T08:40:49,708 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0e1ad619ceb54e96836beb6087ad8bb1 is 175, key is test_row_0/A:col10/1732005647936/Put/seqid=0 2024-11-19T08:40:49,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742031_1207 (size=31255) 2024-11-19T08:40:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T08:40:50,112 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0e1ad619ceb54e96836beb6087ad8bb1 2024-11-19T08:40:50,118 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/6b7b179c6794442788df067557f9a522 is 50, key is test_row_0/B:col10/1732005647936/Put/seqid=0 2024-11-19T08:40:50,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742032_1208 (size=12301) 2024-11-19T08:40:50,522 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/6b7b179c6794442788df067557f9a522 2024-11-19T08:40:50,532 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/d00772d0e12b426a842a58d711e49f13 is 50, key is test_row_0/C:col10/1732005647936/Put/seqid=0 2024-11-19T08:40:50,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742033_1209 (size=12301) 2024-11-19T08:40:50,939 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/d00772d0e12b426a842a58d711e49f13 2024-11-19T08:40:50,945 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/A/0e1ad619ceb54e96836beb6087ad8bb1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0e1ad619ceb54e96836beb6087ad8bb1 2024-11-19T08:40:50,951 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0e1ad619ceb54e96836beb6087ad8bb1, entries=150, sequenceid=353, filesize=30.5 K 2024-11-19T08:40:50,952 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/B/6b7b179c6794442788df067557f9a522 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6b7b179c6794442788df067557f9a522 2024-11-19T08:40:50,956 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6b7b179c6794442788df067557f9a522, entries=150, sequenceid=353, filesize=12.0 K 2024-11-19T08:40:50,957 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/.tmp/C/d00772d0e12b426a842a58d711e49f13 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d00772d0e12b426a842a58d711e49f13 2024-11-19T08:40:50,960 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d00772d0e12b426a842a58d711e49f13, entries=150, sequenceid=353, filesize=12.0 K 2024-11-19T08:40:50,961 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1669ms, sequenceid=353, compaction requested=true 2024-11-19T08:40:50,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/88a6d2a28ae748689793477c1f421ad5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8d75f7ce2ed74d48838e08a29950e8ea, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/92757d2c4e6043409fe4c5ef3b055574, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c9df7e51ccbc4c3c8ab36aaa52711b9c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f730f0dedffb4d028b2c76b5d48cb91a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/7dd798fe465a41c4a335d084d34de998, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/9a67e94053c4431ebf0b20cd510bfed5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/b299b0c045a445e694ddfe37801a19c3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0eafaf6953364508a40e1125e57c1021, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/11906549749a4be6a28fee0156ca639b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f7173d2e39e34551972e4b383130b28e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8a6dffcf69574d65a2b99be4f30b497e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/44562cffe8f6468c9bc9f0983c39f9d9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/77a7376ed1034e3cad8244e3d0ed6e2e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/e49bc10ec40f4696852cf24e1e1c3d1c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/406923731efe4551a631516246703b04, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c10d14f266b341399a0fc13b6df9a61a] to archive 2024-11-19T08:40:50,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:40:50,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/88a6d2a28ae748689793477c1f421ad5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/88a6d2a28ae748689793477c1f421ad5 2024-11-19T08:40:50,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8d75f7ce2ed74d48838e08a29950e8ea to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8d75f7ce2ed74d48838e08a29950e8ea 2024-11-19T08:40:50,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/92757d2c4e6043409fe4c5ef3b055574 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/92757d2c4e6043409fe4c5ef3b055574 2024-11-19T08:40:50,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c9df7e51ccbc4c3c8ab36aaa52711b9c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c9df7e51ccbc4c3c8ab36aaa52711b9c 2024-11-19T08:40:50,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f730f0dedffb4d028b2c76b5d48cb91a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f730f0dedffb4d028b2c76b5d48cb91a 2024-11-19T08:40:50,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/7dd798fe465a41c4a335d084d34de998 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/7dd798fe465a41c4a335d084d34de998 2024-11-19T08:40:50,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/9a67e94053c4431ebf0b20cd510bfed5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/9a67e94053c4431ebf0b20cd510bfed5 2024-11-19T08:40:50,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/b299b0c045a445e694ddfe37801a19c3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/b299b0c045a445e694ddfe37801a19c3 2024-11-19T08:40:50,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0eafaf6953364508a40e1125e57c1021 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0eafaf6953364508a40e1125e57c1021 2024-11-19T08:40:50,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/11906549749a4be6a28fee0156ca639b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/11906549749a4be6a28fee0156ca639b 2024-11-19T08:40:50,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f7173d2e39e34551972e4b383130b28e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/f7173d2e39e34551972e4b383130b28e 2024-11-19T08:40:50,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8a6dffcf69574d65a2b99be4f30b497e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8a6dffcf69574d65a2b99be4f30b497e 2024-11-19T08:40:50,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/44562cffe8f6468c9bc9f0983c39f9d9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/44562cffe8f6468c9bc9f0983c39f9d9 2024-11-19T08:40:50,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/77a7376ed1034e3cad8244e3d0ed6e2e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/77a7376ed1034e3cad8244e3d0ed6e2e 2024-11-19T08:40:50,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/e49bc10ec40f4696852cf24e1e1c3d1c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/e49bc10ec40f4696852cf24e1e1c3d1c 2024-11-19T08:40:50,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/406923731efe4551a631516246703b04 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/406923731efe4551a631516246703b04 2024-11-19T08:40:50,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c10d14f266b341399a0fc13b6df9a61a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/c10d14f266b341399a0fc13b6df9a61a 2024-11-19T08:40:50,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/160297109afc46efa50b404adefae3c3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/9f081ad92047428eb890b3572ac703b0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fffc22e07c974f4c9a9eeed306664ef7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b94357ee6096417c937054bf4d71a06f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/0e63e59b987340fb9c3ff16851b99756, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/634e7f7ecfad4e64bcd8588636184b51, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/13cc135fd6ca4190aa7a4544325d1691, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fba56ac4fc6e4523a64453f95697f7a2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b5a8add9cbce413baeab259d59781ea9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/8f8737d86a504d01b193b001850b9c5a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/34be836c5ef64d20b95a06616495fd5c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/00288a2a771e43ed9ecf2ae5288bd5bb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/5d755d3bf77b4b7eb8dabf7da6749dbf, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/742b37cc3d76449299213c2083771450, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/a92b2c84796d4ace9f304e37147ea761, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2bc8bd2d35a14bb88325fefd51351545, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/75674d4d45d0499a97da09d076b066a3] to archive 2024-11-19T08:40:50,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:40:50,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/160297109afc46efa50b404adefae3c3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/160297109afc46efa50b404adefae3c3 2024-11-19T08:40:50,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/9f081ad92047428eb890b3572ac703b0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/9f081ad92047428eb890b3572ac703b0 2024-11-19T08:40:50,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fffc22e07c974f4c9a9eeed306664ef7 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fffc22e07c974f4c9a9eeed306664ef7 2024-11-19T08:40:50,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b94357ee6096417c937054bf4d71a06f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b94357ee6096417c937054bf4d71a06f 2024-11-19T08:40:50,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/0e63e59b987340fb9c3ff16851b99756 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/0e63e59b987340fb9c3ff16851b99756 2024-11-19T08:40:50,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/634e7f7ecfad4e64bcd8588636184b51 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/634e7f7ecfad4e64bcd8588636184b51 2024-11-19T08:40:50,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/13cc135fd6ca4190aa7a4544325d1691 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/13cc135fd6ca4190aa7a4544325d1691 2024-11-19T08:40:50,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fba56ac4fc6e4523a64453f95697f7a2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/fba56ac4fc6e4523a64453f95697f7a2 2024-11-19T08:40:50,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b5a8add9cbce413baeab259d59781ea9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/b5a8add9cbce413baeab259d59781ea9 2024-11-19T08:40:50,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/8f8737d86a504d01b193b001850b9c5a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/8f8737d86a504d01b193b001850b9c5a 2024-11-19T08:40:50,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/34be836c5ef64d20b95a06616495fd5c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/34be836c5ef64d20b95a06616495fd5c 2024-11-19T08:40:50,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/00288a2a771e43ed9ecf2ae5288bd5bb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/00288a2a771e43ed9ecf2ae5288bd5bb 2024-11-19T08:40:50,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/5d755d3bf77b4b7eb8dabf7da6749dbf to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/5d755d3bf77b4b7eb8dabf7da6749dbf 2024-11-19T08:40:50,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/742b37cc3d76449299213c2083771450 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/742b37cc3d76449299213c2083771450 2024-11-19T08:40:50,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/a92b2c84796d4ace9f304e37147ea761 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/a92b2c84796d4ace9f304e37147ea761 2024-11-19T08:40:50,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2bc8bd2d35a14bb88325fefd51351545 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2bc8bd2d35a14bb88325fefd51351545 2024-11-19T08:40:50,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/75674d4d45d0499a97da09d076b066a3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/75674d4d45d0499a97da09d076b066a3 2024-11-19T08:40:50,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a6ae602b58df47ab990a5f7111bf4199, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/b0452af38538462485e5ecd4e790d167, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/66717e6c1548471ebc64bac04a31c0e3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/c8d314b5445740d3ae734443950d89f9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/ca0e736a179941d4a897538a38d64457, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/89d3285dae62450a99c90668c6fbfa31, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/f34796759b6b49d091d540ea8dbe0fa5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95703440145f40e5883df4a31a67c5ee, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/5bcbe2fc69634751b221bba7f4b86132, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a3a27ae35b064ab3b75b0a7a2e1502df, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/6a3d8be535214a6296a06fb14c297e64, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/df58fb2e1d9f4a5a88ac643f88b28011, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/058db340d88d449b91892ef7f9658c3e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0d6afc006c4247c88296d7435ea80b5c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/4572d8710db04992a7dd53b4468c5d18, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a0c4bfb5cf9345feb85963cdc1c56298, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d703b5377ab04b80ab92ab76a2cdf079] to archive 2024-11-19T08:40:50,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:40:50,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a6ae602b58df47ab990a5f7111bf4199 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a6ae602b58df47ab990a5f7111bf4199 2024-11-19T08:40:50,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/b0452af38538462485e5ecd4e790d167 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/b0452af38538462485e5ecd4e790d167 2024-11-19T08:40:51,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/66717e6c1548471ebc64bac04a31c0e3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/66717e6c1548471ebc64bac04a31c0e3 2024-11-19T08:40:51,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/c8d314b5445740d3ae734443950d89f9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/c8d314b5445740d3ae734443950d89f9 2024-11-19T08:40:51,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/ca0e736a179941d4a897538a38d64457 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/ca0e736a179941d4a897538a38d64457 2024-11-19T08:40:51,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/89d3285dae62450a99c90668c6fbfa31 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/89d3285dae62450a99c90668c6fbfa31 2024-11-19T08:40:51,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/f34796759b6b49d091d540ea8dbe0fa5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/f34796759b6b49d091d540ea8dbe0fa5 2024-11-19T08:40:51,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95703440145f40e5883df4a31a67c5ee to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95703440145f40e5883df4a31a67c5ee 2024-11-19T08:40:51,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/5bcbe2fc69634751b221bba7f4b86132 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/5bcbe2fc69634751b221bba7f4b86132 2024-11-19T08:40:51,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a3a27ae35b064ab3b75b0a7a2e1502df to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a3a27ae35b064ab3b75b0a7a2e1502df 2024-11-19T08:40:51,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/6a3d8be535214a6296a06fb14c297e64 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/6a3d8be535214a6296a06fb14c297e64 2024-11-19T08:40:51,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/df58fb2e1d9f4a5a88ac643f88b28011 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/df58fb2e1d9f4a5a88ac643f88b28011 2024-11-19T08:40:51,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/058db340d88d449b91892ef7f9658c3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/058db340d88d449b91892ef7f9658c3e 2024-11-19T08:40:51,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0d6afc006c4247c88296d7435ea80b5c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0d6afc006c4247c88296d7435ea80b5c 2024-11-19T08:40:51,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/4572d8710db04992a7dd53b4468c5d18 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/4572d8710db04992a7dd53b4468c5d18 2024-11-19T08:40:51,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a0c4bfb5cf9345feb85963cdc1c56298 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/a0c4bfb5cf9345feb85963cdc1c56298 2024-11-19T08:40:51,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d703b5377ab04b80ab92ab76a2cdf079 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d703b5377ab04b80ab92ab76a2cdf079 2024-11-19T08:40:51,016 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/recovered.edits/356.seqid, newMaxSeqId=356, maxSeqId=4 2024-11-19T08:40:51,017 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. 2024-11-19T08:40:51,017 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 392ed962c42d18afe3f39ea7d3a6fd3e: Waiting for close lock at 1732005648104Running coprocessor pre-close hooks at 1732005648104Disabling compacts and flushes for region at 1732005648104Disabling writes for close at 1732005649292 (+1188 ms)Obtaining lock to block concurrent updates at 1732005649292Preparing flush snapshotting stores in 392ed962c42d18afe3f39ea7d3a6fd3e at 1732005649292Finished memstore snapshotting TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732005649292Flushing stores of TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e. at 1732005649293 (+1 ms)Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/A: creating writer at 1732005649293Flushing mob file 392ed962c42d18afe3f39ea7d3a6fd3e/A: appending metadata at 1732005649297 (+4 ms)Flushing mob file 392ed962c42d18afe3f39ea7d3a6fd3e/A: closing flushed file at 1732005649297Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/A: appending metadata at 1732005649707 (+410 ms)Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/A: closing flushed file at 1732005649707Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/B: creating writer at 1732005650116 (+409 ms)Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/B: appending metadata at 1732005650118 (+2 ms)Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/B: closing flushed file at 1732005650118Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/C: creating writer at 1732005650528 (+410 ms)Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/C: appending metadata at 1732005650532 (+4 ms)Flushing 392ed962c42d18afe3f39ea7d3a6fd3e/C: closing flushed file at 1732005650532Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30d54673: reopening flushed file at 1732005650944 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79f662bb: reopening flushed file at 1732005650951 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a7caaa8: reopening flushed file at 1732005650956 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 392ed962c42d18afe3f39ea7d3a6fd3e in 1669ms, sequenceid=353, compaction requested=true at 1732005650961 (+5 ms)Writing region close event to WAL at 1732005651013 (+52 ms)Running coprocessor post-close hooks at 1732005651016 (+3 ms)Closed at 1732005651017 (+1 ms) 2024-11-19T08:40:51,018 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:51,019 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=392ed962c42d18afe3f39ea7d3a6fd3e, regionState=CLOSED 2024-11-19T08:40:51,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:40:51,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T08:40:51,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 392ed962c42d18afe3f39ea7d3a6fd3e, server=3ab37fa97a98,34165,1732005593269 in 3.0690 sec 2024-11-19T08:40:51,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-19T08:40:51,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=392ed962c42d18afe3f39ea7d3a6fd3e, UNASSIGN in 3.0730 sec 2024-11-19T08:40:51,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T08:40:51,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.0770 sec 2024-11-19T08:40:51,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005651027"}]},"ts":"1732005651027"} 2024-11-19T08:40:51,029 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T08:40:51,029 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T08:40:51,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.0880 sec 2024-11-19T08:40:51,524 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T08:40:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T08:40:52,096 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:52,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T08:40:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:52,100 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:52,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T08:40:52,104 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,106 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/recovered.edits] 2024-11-19T08:40:52,109 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0e1ad619ceb54e96836beb6087ad8bb1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/0e1ad619ceb54e96836beb6087ad8bb1 2024-11-19T08:40:52,110 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/140a0a9b99bf4699a09e4840492dce93 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/140a0a9b99bf4699a09e4840492dce93 2024-11-19T08:40:52,112 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/3ec60af824534d5680ab6d951e16f512 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/3ec60af824534d5680ab6d951e16f512 2024-11-19T08:40:52,113 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8c2e97f2af8e42a895734760ba0bee43 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/8c2e97f2af8e42a895734760ba0bee43 2024-11-19T08:40:52,114 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ab956f3fd42d4216ba82fa86f6ff8aab to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ab956f3fd42d4216ba82fa86f6ff8aab 2024-11-19T08:40:52,115 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ed9b4c375b754a138384121b56224666 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/A/ed9b4c375b754a138384121b56224666 2024-11-19T08:40:52,118 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2623ce4b9c57468eb87d868e0373d1b2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/2623ce4b9c57468eb87d868e0373d1b2 2024-11-19T08:40:52,119 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/30665d22a00d41dfa77aa38a275704a7 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/30665d22a00d41dfa77aa38a275704a7 2024-11-19T08:40:52,120 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6b7b179c6794442788df067557f9a522 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6b7b179c6794442788df067557f9a522 2024-11-19T08:40:52,122 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6d853d2184ff469a9b164c1edf2de233 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/6d853d2184ff469a9b164c1edf2de233 2024-11-19T08:40:52,123 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/82f6127237e742e5b32f356d179bd04b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/82f6127237e742e5b32f356d179bd04b 2024-11-19T08:40:52,124 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/efbf576018d34ec3a6f27b21a9b25f07 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/B/efbf576018d34ec3a6f27b21a9b25f07 2024-11-19T08:40:52,127 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0fb8069a39574296994af8468286da32 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/0fb8069a39574296994af8468286da32 2024-11-19T08:40:52,128 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/1867bd00dd6345cdad7e4f7f80191585 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/1867bd00dd6345cdad7e4f7f80191585 2024-11-19T08:40:52,129 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95d5c351908c4bf894a42ad99b20428f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/95d5c351908c4bf894a42ad99b20428f 2024-11-19T08:40:52,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/960b037e4375426c943083b2d254ea6e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/960b037e4375426c943083b2d254ea6e 2024-11-19T08:40:52,132 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d00772d0e12b426a842a58d711e49f13 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/d00772d0e12b426a842a58d711e49f13 2024-11-19T08:40:52,134 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/e269878ce427474db3301be50023023b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/C/e269878ce427474db3301be50023023b 2024-11-19T08:40:52,137 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/recovered.edits/356.seqid to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e/recovered.edits/356.seqid 2024-11-19T08:40:52,138 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,138 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T08:40:52,139 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T08:40:52,139 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T08:40:52,144 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119170e926c30f3441a9dec75ca10ec1b5a_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119170e926c30f3441a9dec75ca10ec1b5a_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,145 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119221a6c152b424d69b75f96ef7b784562_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119221a6c152b424d69b75f96ef7b784562_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,147 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192e36dc8530c04aac9ba56a16764fee18_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192e36dc8530c04aac9ba56a16764fee18_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,148 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fb27473af4b47afaa598272df720cda_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fb27473af4b47afaa598272df720cda_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f5d4488e19b41ba9b2a8aa751fd1e14_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f5d4488e19b41ba9b2a8aa751fd1e14_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,151 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111947c4dc9f2f9d4342ade70a965916c77f_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111947c4dc9f2f9d4342ade70a965916c77f_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,152 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be281738e2248c59adb92908d9fe19e_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be281738e2248c59adb92908d9fe19e_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,153 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194f2fcb33943b409c8b2b6f2f79877557_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194f2fcb33943b409c8b2b6f2f79877557_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,154 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196133564786e245bc8db6a1899c9aa695_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196133564786e245bc8db6a1899c9aa695_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,155 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196bae8b38deac44c18d7e4bdf865949d4_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196bae8b38deac44c18d7e4bdf865949d4_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d43e0895a554622833c0dd60edfabb9_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198d43e0895a554622833c0dd60edfabb9_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,157 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a107960f6b4c4393a0cc4982e999a468_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a107960f6b4c4393a0cc4982e999a468_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a875882617b74520af9afde450d2767d_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a875882617b74520af9afde450d2767d_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,160 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be47999c00634f0cb905fb9c4194d745_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119be47999c00634f0cb905fb9c4194d745_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cce00abf3af340ea87ff41ac3a341fb9_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cce00abf3af340ea87ff41ac3a341fb9_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e13d3f28d11e4c2db781c339dac234ee_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e13d3f28d11e4c2db781c339dac234ee_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,163 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f86a7ab002e0470b8d64bb385d273273_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f86a7ab002e0470b8d64bb385d273273_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f9fca11202604854a1e8150f4a2f17be_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f9fca11202604854a1e8150f4a2f17be_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,166 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fc5f0426560f4d77b66841647d1eb7fa_392ed962c42d18afe3f39ea7d3a6fd3e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fc5f0426560f4d77b66841647d1eb7fa_392ed962c42d18afe3f39ea7d3a6fd3e 2024-11-19T08:40:52,167 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T08:40:52,169 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:52,172 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T08:40:52,174 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T08:40:52,176 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:52,176 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T08:40:52,176 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732005652176"}]},"ts":"9223372036854775807"} 2024-11-19T08:40:52,178 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T08:40:52,178 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 392ed962c42d18afe3f39ea7d3a6fd3e, NAME => 'TestAcidGuarantees,,1732005624378.392ed962c42d18afe3f39ea7d3a6fd3e.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T08:40:52,178 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T08:40:52,178 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732005652178"}]},"ts":"9223372036854775807"} 2024-11-19T08:40:52,180 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T08:40:52,181 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:52,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-19T08:40:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T08:40:52,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T08:40:52,206 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:52,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1816896270_22 at /127.0.0.1:59540 [Waiting for operation #290] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1816896270_22 at /127.0.0.1:42570 [Waiting for operation #460] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1607800269_22 at /127.0.0.1:42694 [Waiting for operation #436] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=482 (was 450) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5443 (was 6509) 2024-11-19T08:40:52,225 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=482, ProcessCount=11, AvailableMemoryMB=5443 2024-11-19T08:40:52,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:40:52,226 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T08:40:52,228 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T08:40:52,229 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:52,229 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-19T08:40:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T08:40:52,229 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T08:40:52,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742034_1210 (size=959) 2024-11-19T08:40:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T08:40:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T08:40:52,637 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:40:52,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742035_1211 (size=53) 2024-11-19T08:40:52,827 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T08:40:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T08:40:53,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:40:53,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 10e67dc5eaf131cd78dc99b8ad946211, disabling compactions & flushes 2024-11-19T08:40:53,043 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:53,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:53,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. after waiting 0 ms 2024-11-19T08:40:53,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:53,043 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:53,043 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 10e67dc5eaf131cd78dc99b8ad946211: Waiting for close lock at 1732005653043Disabling compacts and flushes for region at 1732005653043Disabling writes for close at 1732005653043Writing region close event to WAL at 1732005653043Closed at 1732005653043 2024-11-19T08:40:53,044 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T08:40:53,044 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732005653044"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732005653044"}]},"ts":"1732005653044"} 2024-11-19T08:40:53,046 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T08:40:53,047 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T08:40:53,047 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005653047"}]},"ts":"1732005653047"} 2024-11-19T08:40:53,049 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T08:40:53,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, ASSIGN}] 2024-11-19T08:40:53,051 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, ASSIGN 2024-11-19T08:40:53,052 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, ASSIGN; state=OFFLINE, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=false 2024-11-19T08:40:53,202 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=10e67dc5eaf131cd78dc99b8ad946211, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:53,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, ASSIGN because future has completed 2024-11-19T08:40:53,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:40:53,359 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:53,359 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:40:53,360 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,360 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:40:53,360 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,360 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,362 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,363 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10e67dc5eaf131cd78dc99b8ad946211 columnFamilyName A 2024-11-19T08:40:53,363 DEBUG [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:53,364 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] regionserver.HStore(327): Store=10e67dc5eaf131cd78dc99b8ad946211/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:53,364 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,366 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10e67dc5eaf131cd78dc99b8ad946211 columnFamilyName B 2024-11-19T08:40:53,366 DEBUG [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:53,366 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] regionserver.HStore(327): Store=10e67dc5eaf131cd78dc99b8ad946211/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:53,366 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T08:40:53,368 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10e67dc5eaf131cd78dc99b8ad946211 columnFamilyName C 2024-11-19T08:40:53,368 DEBUG [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:40:53,368 INFO [StoreOpener-10e67dc5eaf131cd78dc99b8ad946211-1 {}] regionserver.HStore(327): Store=10e67dc5eaf131cd78dc99b8ad946211/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:40:53,368 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,369 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,369 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,370 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,370 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,371 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:40:53,372 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,374 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:40:53,374 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 10e67dc5eaf131cd78dc99b8ad946211; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72076912, jitterRate=0.07402968406677246}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:40:53,374 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:53,375 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 10e67dc5eaf131cd78dc99b8ad946211: Running coprocessor pre-open hook at 1732005653360Writing region info on filesystem at 1732005653360Initializing all the Stores at 1732005653361 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005653361Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005653361Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005653361Cleaning up temporary data from old regions at 1732005653370 (+9 ms)Running coprocessor post-open hooks at 1732005653374 (+4 ms)Region opened successfully at 1732005653375 (+1 ms) 2024-11-19T08:40:53,376 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., pid=67, masterSystemTime=1732005653356 2024-11-19T08:40:53,377 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:53,378 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:53,378 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=10e67dc5eaf131cd78dc99b8ad946211, regionState=OPEN, openSeqNum=2, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:53,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:40:53,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T08:40:53,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 in 177 msec 2024-11-19T08:40:53,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-19T08:40:53,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, ASSIGN in 333 msec 2024-11-19T08:40:53,385 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T08:40:53,385 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005653385"}]},"ts":"1732005653385"} 2024-11-19T08:40:53,387 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T08:40:53,388 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T08:40:53,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-19T08:40:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-19T08:40:54,387 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:54,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,391 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b54ca90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,395 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456abfc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,398 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,401 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T08:40:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ab37fa97a98,36325,1732005592414 2024-11-19T08:40:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@808b3ed 2024-11-19T08:40:54,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T08:40:54,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T08:40:54,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4540cfb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77cb4ecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,410 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b732cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,412 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c7a2b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,415 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a71e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59a32097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574144f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e42b0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,420 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9d076b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,421 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a1ed083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,423 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3739385e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34293417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d923957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28af2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,431 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b84d20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,432 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa07b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,434 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a44261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,435 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aea0c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:40:54,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:40:54,437 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519d7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:40:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:40:54,438 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:40:54,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283f31b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:40:54,438 DEBUG [Thread-1018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,439 DEBUG [Thread-1007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,439 DEBUG [Thread-1003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,439 DEBUG [Thread-1012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,439 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:54,439 DEBUG [Thread-1016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,439 DEBUG [Thread-1003 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,440 DEBUG [Thread-1003 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T08:40:54,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T08:40:54,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,441 DEBUG [Thread-1014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,442 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,442 DEBUG [Thread-999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,443 DEBUG [Thread-1010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,443 DEBUG [Thread-1005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:54,443 DEBUG [Thread-1001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:40:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,445 DEBUG [Thread-1010 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,445 DEBUG [Thread-1010 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:40:54,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:40:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:40:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:40:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:54,460 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:54,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T08:40:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005714479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005714480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/95b6282aadee43ca8f597572bb6b6120 is 50, key is test_row_0/A:col10/1732005654457/Put/seqid=0 2024-11-19T08:40:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005714483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005714486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005714487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742036_1212 (size=12001) 2024-11-19T08:40:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T08:40:54,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:54,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005714596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005714596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005714596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005714596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005714596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:54,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:54,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:54,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:54,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T08:40:54,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005714806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005714806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005714806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005714806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005714806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:54,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:54,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:54,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:54,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:54,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:54,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/95b6282aadee43ca8f597572bb6b6120 2024-11-19T08:40:54,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/32f0aed39fc84192842d06f5a21c53cf is 50, key is test_row_0/B:col10/1732005654457/Put/seqid=0 2024-11-19T08:40:54,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742037_1213 (size=12001) 2024-11-19T08:40:54,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/32f0aed39fc84192842d06f5a21c53cf 2024-11-19T08:40:55,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/6ea85ddd0e19499d809a941777748495 is 50, key is test_row_0/C:col10/1732005654457/Put/seqid=0 2024-11-19T08:40:55,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742038_1214 (size=12001) 2024-11-19T08:40:55,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/6ea85ddd0e19499d809a941777748495 2024-11-19T08:40:55,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:55,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:55,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T08:40:55,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/95b6282aadee43ca8f597572bb6b6120 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/95b6282aadee43ca8f597572bb6b6120 2024-11-19T08:40:55,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/95b6282aadee43ca8f597572bb6b6120, entries=150, sequenceid=17, filesize=11.7 K 2024-11-19T08:40:55,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/32f0aed39fc84192842d06f5a21c53cf as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/32f0aed39fc84192842d06f5a21c53cf 2024-11-19T08:40:55,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/32f0aed39fc84192842d06f5a21c53cf, entries=150, sequenceid=17, filesize=11.7 K 2024-11-19T08:40:55,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/6ea85ddd0e19499d809a941777748495 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/6ea85ddd0e19499d809a941777748495 2024-11-19T08:40:55,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/6ea85ddd0e19499d809a941777748495, entries=150, sequenceid=17, filesize=11.7 K 2024-11-19T08:40:55,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 10e67dc5eaf131cd78dc99b8ad946211 in 634ms, sequenceid=17, compaction requested=false 2024-11-19T08:40:55,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:55,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:55,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T08:40:55,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/5cfcf2fde5624a22afa246e3cf3db7b6 is 50, key is test_row_0/A:col10/1732005655118/Put/seqid=0 2024-11-19T08:40:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005715130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005715135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005715135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005715143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005715143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742039_1215 (size=12001) 2024-11-19T08:40:55,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005715246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005715246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005715255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005715256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005715256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005715456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005715456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005715466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005715466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005715466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:55,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/5cfcf2fde5624a22afa246e3cf3db7b6 2024-11-19T08:40:55,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/02eec845682d488a86359a06d8e04d33 is 50, key is test_row_0/B:col10/1732005655118/Put/seqid=0 2024-11-19T08:40:55,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742040_1216 (size=12001) 2024-11-19T08:40:55,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/02eec845682d488a86359a06d8e04d33 2024-11-19T08:40:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T08:40:55,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/946f73f7dde44ad099e2b8d3ab069147 is 50, key is test_row_0/C:col10/1732005655118/Put/seqid=0 2024-11-19T08:40:55,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742041_1217 (size=12001) 2024-11-19T08:40:55,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005715766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005715766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005715777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005715777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005715777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:55,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:55,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:55,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/946f73f7dde44ad099e2b8d3ab069147 2024-11-19T08:40:56,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/5cfcf2fde5624a22afa246e3cf3db7b6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5cfcf2fde5624a22afa246e3cf3db7b6 2024-11-19T08:40:56,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5cfcf2fde5624a22afa246e3cf3db7b6, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T08:40:56,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/02eec845682d488a86359a06d8e04d33 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/02eec845682d488a86359a06d8e04d33 2024-11-19T08:40:56,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/02eec845682d488a86359a06d8e04d33, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T08:40:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/946f73f7dde44ad099e2b8d3ab069147 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/946f73f7dde44ad099e2b8d3ab069147 2024-11-19T08:40:56,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/946f73f7dde44ad099e2b8d3ab069147, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T08:40:56,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 10e67dc5eaf131cd78dc99b8ad946211 in 916ms, sequenceid=41, compaction requested=false 2024-11-19T08:40:56,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:56,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T08:40:56,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:56,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T08:40:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ef1b15b73ad7462993e3d659e5588b1b is 50, key is test_row_0/A:col10/1732005655133/Put/seqid=0 2024-11-19T08:40:56,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742042_1218 (size=12001) 2024-11-19T08:40:56,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005716332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005716334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005716334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005716334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005716335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005716446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005716446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005716446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005716446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005716446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,557 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T08:40:56,584 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ef1b15b73ad7462993e3d659e5588b1b 2024-11-19T08:40:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T08:40:56,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/5f4d3d08a7b446479a94632688f00841 is 50, key is test_row_0/B:col10/1732005655133/Put/seqid=0 2024-11-19T08:40:56,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742043_1219 (size=12001) 2024-11-19T08:40:56,596 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/5f4d3d08a7b446479a94632688f00841 2024-11-19T08:40:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/0d4d9e7a1c0d470db9e37bdb23443018 is 50, key is test_row_0/C:col10/1732005655133/Put/seqid=0 2024-11-19T08:40:56,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742044_1220 (size=12001) 2024-11-19T08:40:56,608 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/0d4d9e7a1c0d470db9e37bdb23443018 2024-11-19T08:40:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ef1b15b73ad7462993e3d659e5588b1b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ef1b15b73ad7462993e3d659e5588b1b 2024-11-19T08:40:56,621 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ef1b15b73ad7462993e3d659e5588b1b, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T08:40:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/5f4d3d08a7b446479a94632688f00841 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/5f4d3d08a7b446479a94632688f00841 2024-11-19T08:40:56,629 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/5f4d3d08a7b446479a94632688f00841, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T08:40:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/0d4d9e7a1c0d470db9e37bdb23443018 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0d4d9e7a1c0d470db9e37bdb23443018 2024-11-19T08:40:56,635 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0d4d9e7a1c0d470db9e37bdb23443018, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T08:40:56,636 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 10e67dc5eaf131cd78dc99b8ad946211 in 497ms, sequenceid=54, compaction requested=true 2024-11-19T08:40:56,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:56,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:56,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T08:40:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T08:40:56,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T08:40:56,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1930 sec 2024-11-19T08:40:56,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.2000 sec 2024-11-19T08:40:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:56,657 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:56,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:40:56,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005716662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005716663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005716664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005716667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005716667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/211a76635417400091b329ffcf72a7dc is 50, key is test_row_0/A:col10/1732005656331/Put/seqid=0 2024-11-19T08:40:56,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742045_1221 (size=14341) 2024-11-19T08:40:56,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005716776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005716776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005716776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005716776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005716975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005716986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005716986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005716986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:56,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005716986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:56,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/211a76635417400091b329ffcf72a7dc 2024-11-19T08:40:57,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/6aa5caabe0e84724b02082bab45aa743 is 50, key is test_row_0/B:col10/1732005656331/Put/seqid=0 2024-11-19T08:40:57,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742046_1222 (size=12001) 2024-11-19T08:40:57,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/6aa5caabe0e84724b02082bab45aa743 2024-11-19T08:40:57,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/b640532053f94beeb6962920ff409d08 is 50, key is test_row_0/C:col10/1732005656331/Put/seqid=0 2024-11-19T08:40:57,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742047_1223 (size=12001) 2024-11-19T08:40:57,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005717296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005717485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/b640532053f94beeb6962920ff409d08 2024-11-19T08:40:57,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/211a76635417400091b329ffcf72a7dc as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/211a76635417400091b329ffcf72a7dc 2024-11-19T08:40:57,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/211a76635417400091b329ffcf72a7dc, entries=200, sequenceid=80, filesize=14.0 K 2024-11-19T08:40:57,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/6aa5caabe0e84724b02082bab45aa743 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6aa5caabe0e84724b02082bab45aa743 2024-11-19T08:40:57,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6aa5caabe0e84724b02082bab45aa743, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T08:40:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/b640532053f94beeb6962920ff409d08 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b640532053f94beeb6962920ff409d08 2024-11-19T08:40:57,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b640532053f94beeb6962920ff409d08, entries=150, sequenceid=80, filesize=11.7 K 2024-11-19T08:40:57,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 10e67dc5eaf131cd78dc99b8ad946211 in 871ms, sequenceid=80, compaction requested=true 2024-11-19T08:40:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:40:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:40:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:57,528 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:40:57,528 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:57,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:57,529 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:57,529 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:57,529 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:40:57,529 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:40:57,530 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:57,530 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:57,530 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/32f0aed39fc84192842d06f5a21c53cf, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/02eec845682d488a86359a06d8e04d33, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/5f4d3d08a7b446479a94632688f00841, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6aa5caabe0e84724b02082bab45aa743] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=46.9 K 2024-11-19T08:40:57,530 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/95b6282aadee43ca8f597572bb6b6120, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5cfcf2fde5624a22afa246e3cf3db7b6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ef1b15b73ad7462993e3d659e5588b1b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/211a76635417400091b329ffcf72a7dc] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=49.2 K 2024-11-19T08:40:57,530 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 32f0aed39fc84192842d06f5a21c53cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732005654453 2024-11-19T08:40:57,530 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95b6282aadee43ca8f597572bb6b6120, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732005654453 2024-11-19T08:40:57,530 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 02eec845682d488a86359a06d8e04d33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732005654479 2024-11-19T08:40:57,530 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cfcf2fde5624a22afa246e3cf3db7b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732005654479 2024-11-19T08:40:57,531 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4d3d08a7b446479a94632688f00841, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005655127 2024-11-19T08:40:57,531 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1b15b73ad7462993e3d659e5588b1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005655127 2024-11-19T08:40:57,531 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aa5caabe0e84724b02082bab45aa743, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732005656331 2024-11-19T08:40:57,531 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 211a76635417400091b329ffcf72a7dc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732005656331 2024-11-19T08:40:57,541 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:57,542 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8828c4b588204bd0b769fc9656c2dd02 is 50, key is test_row_0/B:col10/1732005656331/Put/seqid=0 2024-11-19T08:40:57,546 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:57,546 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/75fa2eaec2b8449183880444700d0468 is 50, key is test_row_0/A:col10/1732005656331/Put/seqid=0 2024-11-19T08:40:57,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742048_1224 (size=12139) 2024-11-19T08:40:57,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742049_1225 (size=12139) 2024-11-19T08:40:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:57,809 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:57,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:40:57,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/054f6d45d9684c6eb5dc348aa1e56d92 is 50, key is test_row_0/A:col10/1732005656666/Put/seqid=0 2024-11-19T08:40:57,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742050_1226 (size=16681) 2024-11-19T08:40:57,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005717833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005717834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005717834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005717837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005717945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005717946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005717946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005717946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:57,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:57,957 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8828c4b588204bd0b769fc9656c2dd02 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8828c4b588204bd0b769fc9656c2dd02 2024-11-19T08:40:57,962 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into 8828c4b588204bd0b769fc9656c2dd02(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:57,962 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:57,962 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=12, startTime=1732005657528; duration=0sec 2024-11-19T08:40:57,962 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:40:57,962 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:40:57,962 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:40:57,963 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:40:57,964 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:40:57,964 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:57,964 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/6ea85ddd0e19499d809a941777748495, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/946f73f7dde44ad099e2b8d3ab069147, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0d4d9e7a1c0d470db9e37bdb23443018, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b640532053f94beeb6962920ff409d08] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=46.9 K 2024-11-19T08:40:57,964 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ea85ddd0e19499d809a941777748495, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732005654453 2024-11-19T08:40:57,965 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 946f73f7dde44ad099e2b8d3ab069147, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732005654479 2024-11-19T08:40:57,966 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d4d9e7a1c0d470db9e37bdb23443018, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732005655127 2024-11-19T08:40:57,966 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b640532053f94beeb6962920ff409d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732005656331 2024-11-19T08:40:57,967 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/75fa2eaec2b8449183880444700d0468 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/75fa2eaec2b8449183880444700d0468 2024-11-19T08:40:57,973 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into 75fa2eaec2b8449183880444700d0468(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:57,973 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:57,973 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=12, startTime=1732005657528; duration=0sec 2024-11-19T08:40:57,973 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:57,974 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:40:57,977 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:40:57,978 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/501ef0a32f6f4272ad7414543a8c4d4d is 50, key is test_row_0/C:col10/1732005656331/Put/seqid=0 2024-11-19T08:40:57,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742051_1227 (size=12139) 2024-11-19T08:40:58,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005718156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005718156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005718156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005718156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/054f6d45d9684c6eb5dc348aa1e56d92 2024-11-19T08:40:58,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/d99c3cb17c85456ca69be4b33a54e9ab is 50, key is test_row_0/B:col10/1732005656666/Put/seqid=0 2024-11-19T08:40:58,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742052_1228 (size=12001) 2024-11-19T08:40:58,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/d99c3cb17c85456ca69be4b33a54e9ab 2024-11-19T08:40:58,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/f0b93feb6f1f4be09ec6e28852a5563a is 50, key is test_row_0/C:col10/1732005656666/Put/seqid=0 2024-11-19T08:40:58,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742053_1229 (size=12001) 2024-11-19T08:40:58,386 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/501ef0a32f6f4272ad7414543a8c4d4d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/501ef0a32f6f4272ad7414543a8c4d4d 2024-11-19T08:40:58,391 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into 501ef0a32f6f4272ad7414543a8c4d4d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:40:58,391 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:58,391 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=12, startTime=1732005657528; duration=0sec 2024-11-19T08:40:58,391 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:40:58,391 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:40:58,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005718466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005718466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005718466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005718467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005718496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:58,499 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:40:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T08:40:58,606 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:58,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T08:40:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T08:40:58,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:58,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:58,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:58,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/f0b93feb6f1f4be09ec6e28852a5563a 2024-11-19T08:40:58,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/054f6d45d9684c6eb5dc348aa1e56d92 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/054f6d45d9684c6eb5dc348aa1e56d92 2024-11-19T08:40:58,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/054f6d45d9684c6eb5dc348aa1e56d92, entries=250, sequenceid=93, filesize=16.3 K 2024-11-19T08:40:58,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/d99c3cb17c85456ca69be4b33a54e9ab as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/d99c3cb17c85456ca69be4b33a54e9ab 2024-11-19T08:40:58,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/d99c3cb17c85456ca69be4b33a54e9ab, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T08:40:58,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/f0b93feb6f1f4be09ec6e28852a5563a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f0b93feb6f1f4be09ec6e28852a5563a 2024-11-19T08:40:58,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f0b93feb6f1f4be09ec6e28852a5563a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T08:40:58,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 10e67dc5eaf131cd78dc99b8ad946211 in 863ms, sequenceid=93, compaction requested=false 2024-11-19T08:40:58,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T08:40:58,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T08:40:58,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:58,761 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:40:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ca3c9cb35888411193f416c19a0692f6 is 50, key is test_row_0/A:col10/1732005657836/Put/seqid=0 2024-11-19T08:40:58,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742054_1230 (size=12001) 2024-11-19T08:40:58,769 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ca3c9cb35888411193f416c19a0692f6 2024-11-19T08:40:58,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/ba8dd83bc34e466fb24ba8fb07e8aeba is 50, key is test_row_0/B:col10/1732005657836/Put/seqid=0 2024-11-19T08:40:58,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742055_1231 (size=12001) 2024-11-19T08:40:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T08:40:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:58,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:59,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005718985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005719014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005719014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005719014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005719115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005719126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005719126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005719126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,180 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/ba8dd83bc34e466fb24ba8fb07e8aeba 2024-11-19T08:40:59,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/b256044dd0524a76b53a9a47a859af7d is 50, key is test_row_0/C:col10/1732005657836/Put/seqid=0 2024-11-19T08:40:59,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742056_1232 (size=12001) 2024-11-19T08:40:59,191 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/b256044dd0524a76b53a9a47a859af7d 2024-11-19T08:40:59,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ca3c9cb35888411193f416c19a0692f6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ca3c9cb35888411193f416c19a0692f6 2024-11-19T08:40:59,199 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ca3c9cb35888411193f416c19a0692f6, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T08:40:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/ba8dd83bc34e466fb24ba8fb07e8aeba as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ba8dd83bc34e466fb24ba8fb07e8aeba 2024-11-19T08:40:59,205 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ba8dd83bc34e466fb24ba8fb07e8aeba, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T08:40:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/b256044dd0524a76b53a9a47a859af7d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b256044dd0524a76b53a9a47a859af7d 2024-11-19T08:40:59,210 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b256044dd0524a76b53a9a47a859af7d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T08:40:59,211 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 10e67dc5eaf131cd78dc99b8ad946211 in 450ms, sequenceid=119, compaction requested=true 2024-11-19T08:40:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:40:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T08:40:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T08:40:59,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T08:40:59,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 603 msec 2024-11-19T08:40:59,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 608 msec 2024-11-19T08:40:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T08:40:59,236 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:40:59,236 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:40:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T08:40:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T08:40:59,238 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:40:59,238 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:40:59,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:40:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:40:59,329 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:40:59,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T08:40:59,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/e641b5b2e04949a28ea6c157918518fd is 50, key is test_row_0/A:col10/1732005659328/Put/seqid=0 2024-11-19T08:40:59,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742057_1233 (size=14541) 2024-11-19T08:40:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T08:40:59,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005719383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:40:59,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:59,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005719389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005719389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005719390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005719496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005719506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005719506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005719506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:40:59,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T08:40:59,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:40:59,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:59,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005719705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005719716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005719716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:40:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005719716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:40:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:40:59,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:40:59,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/e641b5b2e04949a28ea6c157918518fd 2024-11-19T08:40:59,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/6e2b26febfc64c0aa74e9592d62354d8 is 50, key is test_row_0/B:col10/1732005659328/Put/seqid=0 2024-11-19T08:40:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742058_1234 (size=12151) 2024-11-19T08:40:59,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:40:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:40:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:40:59,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:40:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T08:41:00,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:41:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005720016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005720025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005720026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005720026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/6e2b26febfc64c0aa74e9592d62354d8 2024-11-19T08:41:00,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:41:00,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/349b45aebe2b4770a0dde9a007ec9195 is 50, key is test_row_0/C:col10/1732005659328/Put/seqid=0 2024-11-19T08:41:00,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742059_1235 (size=12151) 2024-11-19T08:41:00,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:41:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:00,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T08:41:00,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:41:00,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005720530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005720530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,532 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005720536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005720536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005720536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:00,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/349b45aebe2b4770a0dde9a007ec9195 2024-11-19T08:41:00,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/e641b5b2e04949a28ea6c157918518fd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e641b5b2e04949a28ea6c157918518fd 2024-11-19T08:41:00,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e641b5b2e04949a28ea6c157918518fd, entries=200, sequenceid=134, filesize=14.2 K 2024-11-19T08:41:00,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/6e2b26febfc64c0aa74e9592d62354d8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6e2b26febfc64c0aa74e9592d62354d8 2024-11-19T08:41:00,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6e2b26febfc64c0aa74e9592d62354d8, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T08:41:00,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/349b45aebe2b4770a0dde9a007ec9195 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/349b45aebe2b4770a0dde9a007ec9195 2024-11-19T08:41:00,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/349b45aebe2b4770a0dde9a007ec9195, entries=150, sequenceid=134, filesize=11.9 K 2024-11-19T08:41:00,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 10e67dc5eaf131cd78dc99b8ad946211 in 1259ms, sequenceid=134, compaction requested=true 2024-11-19T08:41:00,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:00,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:00,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:00,588 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:41:00,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:00,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:00,588 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:41:00,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:00,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:00,589 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:41:00,589 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:41:00,589 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:00,589 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:00,589 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,589 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,590 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/75fa2eaec2b8449183880444700d0468, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/054f6d45d9684c6eb5dc348aa1e56d92, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ca3c9cb35888411193f416c19a0692f6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e641b5b2e04949a28ea6c157918518fd] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=54.1 K 2024-11-19T08:41:00,590 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8828c4b588204bd0b769fc9656c2dd02, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/d99c3cb17c85456ca69be4b33a54e9ab, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ba8dd83bc34e466fb24ba8fb07e8aeba, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6e2b26febfc64c0aa74e9592d62354d8] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=47.2 K 2024-11-19T08:41:00,590 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8828c4b588204bd0b769fc9656c2dd02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732005656331 2024-11-19T08:41:00,590 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75fa2eaec2b8449183880444700d0468, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732005656331 2024-11-19T08:41:00,590 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d99c3cb17c85456ca69be4b33a54e9ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732005656659 2024-11-19T08:41:00,590 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054f6d45d9684c6eb5dc348aa1e56d92, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732005656659 2024-11-19T08:41:00,591 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ba8dd83bc34e466fb24ba8fb07e8aeba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732005657832 2024-11-19T08:41:00,591 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca3c9cb35888411193f416c19a0692f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732005657832 2024-11-19T08:41:00,591 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting e641b5b2e04949a28ea6c157918518fd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005658984 2024-11-19T08:41:00,591 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2b26febfc64c0aa74e9592d62354d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005658985 2024-11-19T08:41:00,604 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:00,604 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:00,604 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/d9151ff8820a42b4982a39abb71daaa6 is 50, key is test_row_0/A:col10/1732005659328/Put/seqid=0 2024-11-19T08:41:00,604 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/ff9af08370414e05b0e1c1e750f63123 is 50, key is test_row_0/B:col10/1732005659328/Put/seqid=0 2024-11-19T08:41:00,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742060_1236 (size=12425) 2024-11-19T08:41:00,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742061_1237 (size=12425) 2024-11-19T08:41:00,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T08:41:00,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,615 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T08:41:00,616 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/d9151ff8820a42b4982a39abb71daaa6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d9151ff8820a42b4982a39abb71daaa6 2024-11-19T08:41:00,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/9e56ffe0370e4f80a0fd48e17aa2253e is 50, key is test_row_0/A:col10/1732005659381/Put/seqid=0 2024-11-19T08:41:00,628 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into d9151ff8820a42b4982a39abb71daaa6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:00,628 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:00,628 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=12, startTime=1732005660588; duration=0sec 2024-11-19T08:41:00,628 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:00,628 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:00,628 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:41:00,632 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:41:00,632 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:00,632 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:00,632 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/501ef0a32f6f4272ad7414543a8c4d4d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f0b93feb6f1f4be09ec6e28852a5563a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b256044dd0524a76b53a9a47a859af7d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/349b45aebe2b4770a0dde9a007ec9195] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=47.2 K 2024-11-19T08:41:00,633 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 501ef0a32f6f4272ad7414543a8c4d4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732005656331 2024-11-19T08:41:00,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742062_1238 (size=12151) 2024-11-19T08:41:00,633 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0b93feb6f1f4be09ec6e28852a5563a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732005656659 2024-11-19T08:41:00,633 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting b256044dd0524a76b53a9a47a859af7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732005657832 2024-11-19T08:41:00,633 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/9e56ffe0370e4f80a0fd48e17aa2253e 2024-11-19T08:41:00,634 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 349b45aebe2b4770a0dde9a007ec9195, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005658985 2024-11-19T08:41:00,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/70a4184607c0489996c626e8eb77361a is 50, key is test_row_0/B:col10/1732005659381/Put/seqid=0 2024-11-19T08:41:00,646 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:00,647 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/f8edac0d4500444db062281ca2db94e5 is 50, key is test_row_0/C:col10/1732005659328/Put/seqid=0 2024-11-19T08:41:00,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742064_1240 (size=12425) 2024-11-19T08:41:00,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742063_1239 (size=12151) 2024-11-19T08:41:01,014 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/ff9af08370414e05b0e1c1e750f63123 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ff9af08370414e05b0e1c1e750f63123 2024-11-19T08:41:01,018 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into ff9af08370414e05b0e1c1e750f63123(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:01,018 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:01,018 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=12, startTime=1732005660588; duration=0sec 2024-11-19T08:41:01,018 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:01,018 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:01,052 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/70a4184607c0489996c626e8eb77361a 2024-11-19T08:41:01,058 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/f8edac0d4500444db062281ca2db94e5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f8edac0d4500444db062281ca2db94e5 2024-11-19T08:41:01,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/086c1e7e7df045af9b7b2a80060dae7b is 50, key is test_row_0/C:col10/1732005659381/Put/seqid=0 2024-11-19T08:41:01,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742065_1241 (size=12151) 2024-11-19T08:41:01,065 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into f8edac0d4500444db062281ca2db94e5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:01,065 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:01,065 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=12, startTime=1732005660588; duration=0sec 2024-11-19T08:41:01,065 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:01,065 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T08:41:01,465 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/086c1e7e7df045af9b7b2a80060dae7b 2024-11-19T08:41:01,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/9e56ffe0370e4f80a0fd48e17aa2253e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/9e56ffe0370e4f80a0fd48e17aa2253e 2024-11-19T08:41:01,504 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/9e56ffe0370e4f80a0fd48e17aa2253e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T08:41:01,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/70a4184607c0489996c626e8eb77361a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/70a4184607c0489996c626e8eb77361a 2024-11-19T08:41:01,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/70a4184607c0489996c626e8eb77361a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T08:41:01,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/086c1e7e7df045af9b7b2a80060dae7b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/086c1e7e7df045af9b7b2a80060dae7b 2024-11-19T08:41:01,514 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/086c1e7e7df045af9b7b2a80060dae7b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T08:41:01,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 10e67dc5eaf131cd78dc99b8ad946211 in 900ms, sequenceid=156, compaction requested=false 2024-11-19T08:41:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:01,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T08:41:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T08:41:01,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T08:41:01,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2770 sec 2024-11-19T08:41:01,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.2820 sec 2024-11-19T08:41:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:01,558 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:01,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:41:01,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/5099c0e5b8f74c558d2cfcfdb993077e is 50, key is test_row_0/A:col10/1732005661557/Put/seqid=0 2024-11-19T08:41:01,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742066_1242 (size=14541) 2024-11-19T08:41:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005721587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005721588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005721588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005721589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005721696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005721696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005721696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005721705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005721906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005721906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005721906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005721916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:01,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/5099c0e5b8f74c558d2cfcfdb993077e 2024-11-19T08:41:01,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8486fec2829541b5bc86907534164efc is 50, key is test_row_0/B:col10/1732005661557/Put/seqid=0 2024-11-19T08:41:01,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742067_1243 (size=12151) 2024-11-19T08:41:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005722215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005722216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005722216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005722226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8486fec2829541b5bc86907534164efc 2024-11-19T08:41:02,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/9a689f00963841a39949ef5a04f16535 is 50, key is test_row_0/C:col10/1732005661557/Put/seqid=0 2024-11-19T08:41:02,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742068_1244 (size=12151) 2024-11-19T08:41:02,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005722725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005722726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005722726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005722736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:02,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/9a689f00963841a39949ef5a04f16535 2024-11-19T08:41:02,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/5099c0e5b8f74c558d2cfcfdb993077e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5099c0e5b8f74c558d2cfcfdb993077e 2024-11-19T08:41:02,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5099c0e5b8f74c558d2cfcfdb993077e, entries=200, sequenceid=171, filesize=14.2 K 2024-11-19T08:41:02,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8486fec2829541b5bc86907534164efc as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8486fec2829541b5bc86907534164efc 2024-11-19T08:41:02,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8486fec2829541b5bc86907534164efc, entries=150, sequenceid=171, filesize=11.9 K 2024-11-19T08:41:02,817 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T08:41:02,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/9a689f00963841a39949ef5a04f16535 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9a689f00963841a39949ef5a04f16535 2024-11-19T08:41:02,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9a689f00963841a39949ef5a04f16535, entries=150, sequenceid=171, filesize=11.9 K 2024-11-19T08:41:02,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 10e67dc5eaf131cd78dc99b8ad946211 in 1264ms, sequenceid=171, compaction requested=true 2024-11-19T08:41:02,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:02,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:02,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:02,822 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:02,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:02,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:02,823 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:02,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:02,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:02,824 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:02,824 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:02,824 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:02,824 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:02,824 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:02,824 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:02,824 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ff9af08370414e05b0e1c1e750f63123, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/70a4184607c0489996c626e8eb77361a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8486fec2829541b5bc86907534164efc] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=35.9 K 2024-11-19T08:41:02,824 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d9151ff8820a42b4982a39abb71daaa6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/9e56ffe0370e4f80a0fd48e17aa2253e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5099c0e5b8f74c558d2cfcfdb993077e] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=38.2 K 2024-11-19T08:41:02,824 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9151ff8820a42b4982a39abb71daaa6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005658985 2024-11-19T08:41:02,824 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ff9af08370414e05b0e1c1e750f63123, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005658985 2024-11-19T08:41:02,825 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a4184607c0489996c626e8eb77361a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732005659381 2024-11-19T08:41:02,825 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e56ffe0370e4f80a0fd48e17aa2253e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732005659381 2024-11-19T08:41:02,825 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5099c0e5b8f74c558d2cfcfdb993077e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732005661547 2024-11-19T08:41:02,825 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8486fec2829541b5bc86907534164efc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732005661547 2024-11-19T08:41:02,828 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T08:41:02,828 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T08:41:02,834 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:02,834 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:02,835 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/1691ca70a8d84762b9f3f5ca5dad811a is 50, key is test_row_0/A:col10/1732005661557/Put/seqid=0 2024-11-19T08:41:02,835 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/fceb1d8764d44a6aab2903151e0f01dd is 50, key is test_row_0/B:col10/1732005661557/Put/seqid=0 2024-11-19T08:41:02,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742070_1246 (size=12527) 2024-11-19T08:41:02,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742069_1245 (size=12527) 2024-11-19T08:41:02,854 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/fceb1d8764d44a6aab2903151e0f01dd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/fceb1d8764d44a6aab2903151e0f01dd 2024-11-19T08:41:02,855 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/1691ca70a8d84762b9f3f5ca5dad811a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/1691ca70a8d84762b9f3f5ca5dad811a 2024-11-19T08:41:02,861 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into fceb1d8764d44a6aab2903151e0f01dd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:02,861 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into 1691ca70a8d84762b9f3f5ca5dad811a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:02,861 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:02,861 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:02,861 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=13, startTime=1732005662822; duration=0sec 2024-11-19T08:41:02,861 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=13, startTime=1732005662823; duration=0sec 2024-11-19T08:41:02,861 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:02,861 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:02,861 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:02,861 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:02,861 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:02,862 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:02,862 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:02,862 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:02,862 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f8edac0d4500444db062281ca2db94e5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/086c1e7e7df045af9b7b2a80060dae7b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9a689f00963841a39949ef5a04f16535] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=35.9 K 2024-11-19T08:41:02,863 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f8edac0d4500444db062281ca2db94e5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732005658985 2024-11-19T08:41:02,863 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 086c1e7e7df045af9b7b2a80060dae7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732005659381 2024-11-19T08:41:02,864 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a689f00963841a39949ef5a04f16535, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732005661547 2024-11-19T08:41:02,869 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:02,870 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/ba23da8089354f2d9b6c139a69b59df9 is 50, key is test_row_0/C:col10/1732005661557/Put/seqid=0 2024-11-19T08:41:02,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742071_1247 (size=12527) 2024-11-19T08:41:03,281 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/ba23da8089354f2d9b6c139a69b59df9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/ba23da8089354f2d9b6c139a69b59df9 2024-11-19T08:41:03,285 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into ba23da8089354f2d9b6c139a69b59df9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:03,285 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:03,285 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=13, startTime=1732005662823; duration=0sec 2024-11-19T08:41:03,285 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:03,285 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T08:41:03,396 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:03,397 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T08:41:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T08:41:03,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:03,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:03,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T08:41:03,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T08:41:03,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:03,551 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:41:03,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/af2f9b9a4f444101a818a201b23ccab0 is 50, key is test_row_0/A:col10/1732005661587/Put/seqid=0 2024-11-19T08:41:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742072_1248 (size=12151) 2024-11-19T08:41:03,571 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/af2f9b9a4f444101a818a201b23ccab0 2024-11-19T08:41:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/28271a6783f9431698010f56a673da99 is 50, key is test_row_0/B:col10/1732005661587/Put/seqid=0 2024-11-19T08:41:03,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742073_1249 (size=12151) 2024-11-19T08:41:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T08:41:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:03,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:03,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005723747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005723747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005723751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005723752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005723856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005723856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005723866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005723866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:03,992 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/28271a6783f9431698010f56a673da99 2024-11-19T08:41:03,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/4310c3307e9e4208b5bbd4ae2749f5a2 is 50, key is test_row_0/C:col10/1732005661587/Put/seqid=0 2024-11-19T08:41:04,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742074_1250 (size=12151) 2024-11-19T08:41:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T08:41:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005724066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005724066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005724075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005724075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005724376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005724376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005724386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005724386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,404 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/4310c3307e9e4208b5bbd4ae2749f5a2 2024-11-19T08:41:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/af2f9b9a4f444101a818a201b23ccab0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/af2f9b9a4f444101a818a201b23ccab0 2024-11-19T08:41:04,413 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/af2f9b9a4f444101a818a201b23ccab0, entries=150, sequenceid=199, filesize=11.9 K 2024-11-19T08:41:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/28271a6783f9431698010f56a673da99 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/28271a6783f9431698010f56a673da99 2024-11-19T08:41:04,420 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/28271a6783f9431698010f56a673da99, entries=150, sequenceid=199, filesize=11.9 K 2024-11-19T08:41:04,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/4310c3307e9e4208b5bbd4ae2749f5a2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/4310c3307e9e4208b5bbd4ae2749f5a2 2024-11-19T08:41:04,425 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/4310c3307e9e4208b5bbd4ae2749f5a2, entries=150, sequenceid=199, filesize=11.9 K 2024-11-19T08:41:04,426 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 10e67dc5eaf131cd78dc99b8ad946211 in 875ms, sequenceid=199, compaction requested=false 2024-11-19T08:41:04,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:04,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T08:41:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T08:41:04,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T08:41:04,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0300 sec 2024-11-19T08:41:04,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0370 sec 2024-11-19T08:41:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T08:41:04,536 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:04,536 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T08:41:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T08:41:04,538 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:04,539 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:04,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:04,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:04,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:41:04,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/d64dd338f76c4200a8ef1d4f61dd3ed4 is 50, key is test_row_0/A:col10/1732005663750/Put/seqid=0 2024-11-19T08:41:04,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742075_1251 (size=9757) 2024-11-19T08:41:04,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/d64dd338f76c4200a8ef1d4f61dd3ed4 2024-11-19T08:41:04,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/91826b13e873453ebcc5796ddad68f2e is 50, key is test_row_0/B:col10/1732005663750/Put/seqid=0 2024-11-19T08:41:04,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742076_1252 (size=9757) 2024-11-19T08:41:04,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/91826b13e873453ebcc5796ddad68f2e 2024-11-19T08:41:04,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/618cfbef767248b4a89821267a203313 is 50, key is test_row_0/C:col10/1732005663750/Put/seqid=0 2024-11-19T08:41:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T08:41:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742077_1253 (size=9757) 2024-11-19T08:41:04,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005724662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T08:41:04,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:04,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:04,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:04,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:04,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005724776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T08:41:04,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:04,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:04,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:04,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:04,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T08:41:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005724886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005724886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005724896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005724896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005724986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T08:41:05,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:05,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/618cfbef767248b4a89821267a203313 2024-11-19T08:41:05,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/d64dd338f76c4200a8ef1d4f61dd3ed4 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d64dd338f76c4200a8ef1d4f61dd3ed4 2024-11-19T08:41:05,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d64dd338f76c4200a8ef1d4f61dd3ed4, entries=100, sequenceid=211, filesize=9.5 K 2024-11-19T08:41:05,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/91826b13e873453ebcc5796ddad68f2e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/91826b13e873453ebcc5796ddad68f2e 2024-11-19T08:41:05,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/91826b13e873453ebcc5796ddad68f2e, entries=100, sequenceid=211, filesize=9.5 K 2024-11-19T08:41:05,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/618cfbef767248b4a89821267a203313 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/618cfbef767248b4a89821267a203313 2024-11-19T08:41:05,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/618cfbef767248b4a89821267a203313, entries=100, sequenceid=211, filesize=9.5 K 2024-11-19T08:41:05,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 10e67dc5eaf131cd78dc99b8ad946211 in 545ms, sequenceid=211, compaction requested=true 2024-11-19T08:41:05,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:05,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:05,096 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:05,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:05,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:05,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:05,096 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:05,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:05,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:05,097 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:05,097 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:05,097 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:05,097 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:05,097 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,097 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,097 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/fceb1d8764d44a6aab2903151e0f01dd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/28271a6783f9431698010f56a673da99, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/91826b13e873453ebcc5796ddad68f2e] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=33.6 K 2024-11-19T08:41:05,097 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/1691ca70a8d84762b9f3f5ca5dad811a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/af2f9b9a4f444101a818a201b23ccab0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d64dd338f76c4200a8ef1d4f61dd3ed4] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=33.6 K 2024-11-19T08:41:05,097 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting fceb1d8764d44a6aab2903151e0f01dd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732005661547 2024-11-19T08:41:05,097 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1691ca70a8d84762b9f3f5ca5dad811a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732005661547 2024-11-19T08:41:05,098 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 28271a6783f9431698010f56a673da99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732005661580 2024-11-19T08:41:05,098 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting af2f9b9a4f444101a818a201b23ccab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732005661580 2024-11-19T08:41:05,098 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 91826b13e873453ebcc5796ddad68f2e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732005663750 2024-11-19T08:41:05,098 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64dd338f76c4200a8ef1d4f61dd3ed4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732005663750 2024-11-19T08:41:05,118 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:05,119 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/fe58281439404993aa8b7ea36ee2b766 is 50, key is test_row_0/A:col10/1732005663750/Put/seqid=0 2024-11-19T08:41:05,123 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:05,124 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/011098714c8146c2acfbc3e78b66b3dd is 50, key is test_row_0/B:col10/1732005663750/Put/seqid=0 2024-11-19T08:41:05,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T08:41:05,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T08:41:05,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742079_1255 (size=12629) 2024-11-19T08:41:05,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742078_1254 (size=12629) 2024-11-19T08:41:05,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/7f79a2a664824a2e928a50058fdcb9a4 is 50, key is test_row_0/A:col10/1732005664649/Put/seqid=0 2024-11-19T08:41:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T08:41:05,171 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/011098714c8146c2acfbc3e78b66b3dd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/011098714c8146c2acfbc3e78b66b3dd 2024-11-19T08:41:05,173 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/fe58281439404993aa8b7ea36ee2b766 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/fe58281439404993aa8b7ea36ee2b766 2024-11-19T08:41:05,180 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into 011098714c8146c2acfbc3e78b66b3dd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:05,180 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:05,180 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=13, startTime=1732005665096; duration=0sec 2024-11-19T08:41:05,180 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:05,180 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:05,181 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:05,183 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into fe58281439404993aa8b7ea36ee2b766(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:05,183 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:05,183 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=13, startTime=1732005665095; duration=0sec 2024-11-19T08:41:05,183 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:05,183 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:05,183 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:05,183 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:05,183 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,183 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/ba23da8089354f2d9b6c139a69b59df9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/4310c3307e9e4208b5bbd4ae2749f5a2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/618cfbef767248b4a89821267a203313] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=33.6 K 2024-11-19T08:41:05,184 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ba23da8089354f2d9b6c139a69b59df9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732005661547 2024-11-19T08:41:05,184 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4310c3307e9e4208b5bbd4ae2749f5a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732005661580 2024-11-19T08:41:05,185 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 618cfbef767248b4a89821267a203313, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732005663750 2024-11-19T08:41:05,218 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:05,218 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/c2c9d2912e0c457ca619117ee24af431 is 50, key is test_row_0/C:col10/1732005663750/Put/seqid=0 2024-11-19T08:41:05,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742080_1256 (size=12151) 2024-11-19T08:41:05,219 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/7f79a2a664824a2e928a50058fdcb9a4 2024-11-19T08:41:05,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/b7d3b2573cfb49acb6196f6f00b76d8d is 50, key is test_row_0/B:col10/1732005664649/Put/seqid=0 2024-11-19T08:41:05,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742081_1257 (size=12629) 2024-11-19T08:41:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742082_1258 (size=12151) 2024-11-19T08:41:05,290 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/b7d3b2573cfb49acb6196f6f00b76d8d 2024-11-19T08:41:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:05,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/d4ac2509c6994d119ceda731b1c71a93 is 50, key is test_row_0/C:col10/1732005664649/Put/seqid=0 2024-11-19T08:41:05,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742083_1259 (size=12151) 2024-11-19T08:41:05,326 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/d4ac2509c6994d119ceda731b1c71a93 2024-11-19T08:41:05,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/7f79a2a664824a2e928a50058fdcb9a4 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/7f79a2a664824a2e928a50058fdcb9a4 2024-11-19T08:41:05,338 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/7f79a2a664824a2e928a50058fdcb9a4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T08:41:05,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/b7d3b2573cfb49acb6196f6f00b76d8d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b7d3b2573cfb49acb6196f6f00b76d8d 2024-11-19T08:41:05,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b7d3b2573cfb49acb6196f6f00b76d8d, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T08:41:05,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/d4ac2509c6994d119ceda731b1c71a93 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/d4ac2509c6994d119ceda731b1c71a93 2024-11-19T08:41:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005725342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,351 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/d4ac2509c6994d119ceda731b1c71a93, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T08:41:05,353 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 10e67dc5eaf131cd78dc99b8ad946211 in 198ms, sequenceid=235, compaction requested=false 2024-11-19T08:41:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T08:41:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T08:41:05,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T08:41:05,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 815 msec 2024-11-19T08:41:05,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 824 msec 2024-11-19T08:41:05,460 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:05,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T08:41:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:05,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/3ad94a6d14194443926b941b9452bf56 is 50, key is test_row_0/A:col10/1732005665334/Put/seqid=0 2024-11-19T08:41:05,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742084_1260 (size=14541) 2024-11-19T08:41:05,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005725599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,656 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/c2c9d2912e0c457ca619117ee24af431 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c2c9d2912e0c457ca619117ee24af431 2024-11-19T08:41:05,663 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into c2c9d2912e0c457ca619117ee24af431(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:05,663 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:05,664 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=13, startTime=1732005665096; duration=0sec 2024-11-19T08:41:05,664 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:05,664 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T08:41:05,677 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:05,678 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T08:41:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T08:41:05,681 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:05,683 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:05,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:05,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005725716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T08:41:05,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:05,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:05,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/3ad94a6d14194443926b941b9452bf56 2024-11-19T08:41:05,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005725896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,898 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005725906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005725906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,909 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005725906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,910 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,910 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/2999391c738944fe92c7788a1fc56970 is 50, key is test_row_0/B:col10/1732005665334/Put/seqid=0 2024-11-19T08:41:05,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742085_1261 (size=12151) 2024-11-19T08:41:05,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005725925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:05,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:05,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T08:41:06,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:06,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:06,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005726235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:06,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T08:41:06,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/2999391c738944fe92c7788a1fc56970 2024-11-19T08:41:06,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/a2d8b815f57841dd9cd945a5d2f08104 is 50, key is test_row_0/C:col10/1732005665334/Put/seqid=0 2024-11-19T08:41:06,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742086_1262 (size=12151) 2024-11-19T08:41:06,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:06,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:06,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:06,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:06,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005726746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:06,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/a2d8b815f57841dd9cd945a5d2f08104 2024-11-19T08:41:06,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/3ad94a6d14194443926b941b9452bf56 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/3ad94a6d14194443926b941b9452bf56 2024-11-19T08:41:06,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:06,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/3ad94a6d14194443926b941b9452bf56, entries=200, sequenceid=250, filesize=14.2 K 2024-11-19T08:41:06,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/2999391c738944fe92c7788a1fc56970 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/2999391c738944fe92c7788a1fc56970 2024-11-19T08:41:06,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/2999391c738944fe92c7788a1fc56970, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T08:41:06,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/a2d8b815f57841dd9cd945a5d2f08104 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/a2d8b815f57841dd9cd945a5d2f08104 2024-11-19T08:41:06,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/a2d8b815f57841dd9cd945a5d2f08104, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T08:41:06,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 10e67dc5eaf131cd78dc99b8ad946211 in 1312ms, sequenceid=250, compaction requested=true 2024-11-19T08:41:06,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:06,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:06,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:06,772 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:06,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:06,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:06,772 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:06,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:06,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:06,774 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:06,774 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:06,774 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:06,774 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:06,774 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,774 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,774 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/fe58281439404993aa8b7ea36ee2b766, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/7f79a2a664824a2e928a50058fdcb9a4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/3ad94a6d14194443926b941b9452bf56] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=38.4 K 2024-11-19T08:41:06,774 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/011098714c8146c2acfbc3e78b66b3dd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b7d3b2573cfb49acb6196f6f00b76d8d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/2999391c738944fe92c7788a1fc56970] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.1 K 2024-11-19T08:41:06,775 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 011098714c8146c2acfbc3e78b66b3dd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732005661587 2024-11-19T08:41:06,775 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe58281439404993aa8b7ea36ee2b766, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732005661587 2024-11-19T08:41:06,775 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d3b2573cfb49acb6196f6f00b76d8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732005664649 2024-11-19T08:41:06,775 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f79a2a664824a2e928a50058fdcb9a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732005664649 2024-11-19T08:41:06,776 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 2999391c738944fe92c7788a1fc56970, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732005665302 2024-11-19T08:41:06,776 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ad94a6d14194443926b941b9452bf56, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732005665302 2024-11-19T08:41:06,785 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:06,786 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ae416a0e223c4ff58b709fd8ae8b93cd is 50, key is test_row_0/A:col10/1732005665334/Put/seqid=0 2024-11-19T08:41:06,788 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:06,788 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/63a80568e8044e92b979c4f77dd5d7bc is 50, key is test_row_0/B:col10/1732005665334/Put/seqid=0 2024-11-19T08:41:06,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742087_1263 (size=12731) 2024-11-19T08:41:06,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742088_1264 (size=12731) 2024-11-19T08:41:06,810 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/63a80568e8044e92b979c4f77dd5d7bc as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/63a80568e8044e92b979c4f77dd5d7bc 2024-11-19T08:41:06,815 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into 63a80568e8044e92b979c4f77dd5d7bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:06,815 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:06,815 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=13, startTime=1732005666772; duration=0sec 2024-11-19T08:41:06,816 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:06,816 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:06,816 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T08:41:06,817 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:06,817 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:06,817 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,817 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c2c9d2912e0c457ca619117ee24af431, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/d4ac2509c6994d119ceda731b1c71a93, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/a2d8b815f57841dd9cd945a5d2f08104] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.1 K 2024-11-19T08:41:06,818 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c9d2912e0c457ca619117ee24af431, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732005661587 2024-11-19T08:41:06,818 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ac2509c6994d119ceda731b1c71a93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732005664649 2024-11-19T08:41:06,818 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d8b815f57841dd9cd945a5d2f08104, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732005665302 2024-11-19T08:41:06,825 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:06,825 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/39921a3e75c84b859f3045b31c1cf612 is 50, key is test_row_0/C:col10/1732005665334/Put/seqid=0 2024-11-19T08:41:06,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742089_1265 (size=12731) 2024-11-19T08:41:06,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T08:41:06,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:06,910 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:41:06,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/06f3cfa3403543b7a699cae30f46be86 is 50, key is test_row_0/A:col10/1732005665597/Put/seqid=0 2024-11-19T08:41:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742090_1266 (size=12301) 2024-11-19T08:41:07,197 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ae416a0e223c4ff58b709fd8ae8b93cd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ae416a0e223c4ff58b709fd8ae8b93cd 2024-11-19T08:41:07,201 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into ae416a0e223c4ff58b709fd8ae8b93cd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:07,201 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:07,201 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=13, startTime=1732005666772; duration=0sec 2024-11-19T08:41:07,201 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:07,201 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:07,235 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/39921a3e75c84b859f3045b31c1cf612 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/39921a3e75c84b859f3045b31c1cf612 2024-11-19T08:41:07,240 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into 39921a3e75c84b859f3045b31c1cf612(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:07,240 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:07,240 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=13, startTime=1732005666772; duration=0sec 2024-11-19T08:41:07,240 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:07,240 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:07,319 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/06f3cfa3403543b7a699cae30f46be86 2024-11-19T08:41:07,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/150e6df3b91f452a9d50e1955b749068 is 50, key is test_row_0/B:col10/1732005665597/Put/seqid=0 2024-11-19T08:41:07,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742091_1267 (size=12301) 2024-11-19T08:41:07,733 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/150e6df3b91f452a9d50e1955b749068 2024-11-19T08:41:07,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/2c610d5dbec341d49f73d206ab958105 is 50, key is test_row_0/C:col10/1732005665597/Put/seqid=0 2024-11-19T08:41:07,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742092_1268 (size=12301) 2024-11-19T08:41:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:07,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:07,769 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/2c610d5dbec341d49f73d206ab958105 2024-11-19T08:41:07,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/06f3cfa3403543b7a699cae30f46be86 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/06f3cfa3403543b7a699cae30f46be86 2024-11-19T08:41:07,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/06f3cfa3403543b7a699cae30f46be86, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T08:41:07,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/150e6df3b91f452a9d50e1955b749068 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/150e6df3b91f452a9d50e1955b749068 2024-11-19T08:41:07,783 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/150e6df3b91f452a9d50e1955b749068, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T08:41:07,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/2c610d5dbec341d49f73d206ab958105 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/2c610d5dbec341d49f73d206ab958105 2024-11-19T08:41:07,789 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/2c610d5dbec341d49f73d206ab958105, entries=150, sequenceid=275, filesize=12.0 K 2024-11-19T08:41:07,790 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=46.96 KB/48090 for 10e67dc5eaf131cd78dc99b8ad946211 in 879ms, sequenceid=275, compaction requested=false 2024-11-19T08:41:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T08:41:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T08:41:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:07,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T08:41:07,793 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:07,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1080 sec 2024-11-19T08:41:07,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:41:07,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.1160 sec 2024-11-19T08:41:07,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/0f657865eafb46ddab28db507e51c72d is 50, key is test_row_0/A:col10/1732005667775/Put/seqid=0 2024-11-19T08:41:07,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742093_1269 (size=14741) 2024-11-19T08:41:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T08:41:07,826 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:07,827 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T08:41:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T08:41:07,828 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:07,829 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:07,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005727884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:07,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005727916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:07,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005727916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:07,918 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,918 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:07,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:07,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005727926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:07,928 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T08:41:07,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005727936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:07,939 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 10e67dc5eaf131cd78dc99b8ad946211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:07,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:07,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:07,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:07,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:07,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:07,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:07,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005727996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:07,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:08,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T08:41:08,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/0f657865eafb46ddab28db507e51c72d 2024-11-19T08:41:08,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/b3e8ca4ce20345f894287bbc6b07960d is 50, key is test_row_0/B:col10/1732005667775/Put/seqid=0 2024-11-19T08:41:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005728206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:08,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742094_1270 (size=12301) 2024-11-19T08:41:08,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:08,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T08:41:08,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005728516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:08,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/b3e8ca4ce20345f894287bbc6b07960d 2024-11-19T08:41:08,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/927ee11d909f4a5e97aaaac8b07d1a03 is 50, key is test_row_0/C:col10/1732005667775/Put/seqid=0 2024-11-19T08:41:08,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742095_1271 (size=12301) 2024-11-19T08:41:08,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:08,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:08,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:08,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:08,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:08,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T08:41:09,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/927ee11d909f4a5e97aaaac8b07d1a03 2024-11-19T08:41:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/0f657865eafb46ddab28db507e51c72d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/0f657865eafb46ddab28db507e51c72d 2024-11-19T08:41:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005729026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:09,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/0f657865eafb46ddab28db507e51c72d, entries=200, sequenceid=288, filesize=14.4 K 2024-11-19T08:41:09,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/b3e8ca4ce20345f894287bbc6b07960d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b3e8ca4ce20345f894287bbc6b07960d 2024-11-19T08:41:09,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b3e8ca4ce20345f894287bbc6b07960d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T08:41:09,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/927ee11d909f4a5e97aaaac8b07d1a03 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/927ee11d909f4a5e97aaaac8b07d1a03 2024-11-19T08:41:09,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/927ee11d909f4a5e97aaaac8b07d1a03, entries=150, sequenceid=288, filesize=12.0 K 2024-11-19T08:41:09,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 10e67dc5eaf131cd78dc99b8ad946211 in 1246ms, sequenceid=288, compaction requested=true 2024-11-19T08:41:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:09,040 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:09,040 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:09,041 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:09,041 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:09,041 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/63a80568e8044e92b979c4f77dd5d7bc, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/150e6df3b91f452a9d50e1955b749068, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b3e8ca4ce20345f894287bbc6b07960d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.5 K 2024-11-19T08:41:09,041 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ae416a0e223c4ff58b709fd8ae8b93cd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/06f3cfa3403543b7a699cae30f46be86, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/0f657865eafb46ddab28db507e51c72d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=38.8 K 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a80568e8044e92b979c4f77dd5d7bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732005665302 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae416a0e223c4ff58b709fd8ae8b93cd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732005665302 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 150e6df3b91f452a9d50e1955b749068, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732005665571 2024-11-19T08:41:09,041 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f3cfa3403543b7a699cae30f46be86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732005665571 2024-11-19T08:41:09,042 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e8ca4ce20345f894287bbc6b07960d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732005667775 2024-11-19T08:41:09,042 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f657865eafb46ddab28db507e51c72d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732005667775 2024-11-19T08:41:09,048 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:09,048 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/a37dbe2b809142349f837e99451614a3 is 50, key is test_row_0/A:col10/1732005667775/Put/seqid=0 2024-11-19T08:41:09,050 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:09,051 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/61358f1c05084309b6550e2f8826779d is 50, key is test_row_0/B:col10/1732005667775/Put/seqid=0 2024-11-19T08:41:09,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T08:41:09,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:09,052 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:41:09,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742097_1273 (size=12983) 2024-11-19T08:41:09,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742096_1272 (size=12983) 2024-11-19T08:41:09,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/2cb8e390680e41cc8fc2ab56c2f7d795 is 50, key is test_row_0/A:col10/1732005667847/Put/seqid=0 2024-11-19T08:41:09,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742098_1274 (size=12301) 2024-11-19T08:41:09,461 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/61358f1c05084309b6550e2f8826779d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/61358f1c05084309b6550e2f8826779d 2024-11-19T08:41:09,463 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/a37dbe2b809142349f837e99451614a3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a37dbe2b809142349f837e99451614a3 2024-11-19T08:41:09,465 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into 61358f1c05084309b6550e2f8826779d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:09,465 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:09,465 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=13, startTime=1732005669040; duration=0sec 2024-11-19T08:41:09,465 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:09,465 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:09,465 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:09,468 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:09,468 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:09,468 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:09,468 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/39921a3e75c84b859f3045b31c1cf612, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/2c610d5dbec341d49f73d206ab958105, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/927ee11d909f4a5e97aaaac8b07d1a03] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.5 K 2024-11-19T08:41:09,469 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 39921a3e75c84b859f3045b31c1cf612, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732005665302 2024-11-19T08:41:09,469 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c610d5dbec341d49f73d206ab958105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732005665571 2024-11-19T08:41:09,469 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/2cb8e390680e41cc8fc2ab56c2f7d795 2024-11-19T08:41:09,469 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 927ee11d909f4a5e97aaaac8b07d1a03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732005667775 2024-11-19T08:41:09,469 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into a37dbe2b809142349f837e99451614a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:09,469 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:09,469 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=13, startTime=1732005669040; duration=0sec 2024-11-19T08:41:09,470 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:09,470 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:09,475 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:09,476 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/09f09ef7a2774951a8ee697aaca3d1cd is 50, key is test_row_0/C:col10/1732005667775/Put/seqid=0 2024-11-19T08:41:09,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/7f7e9497a18c40578d99f7f42907e099 is 50, key is test_row_0/B:col10/1732005667847/Put/seqid=0 2024-11-19T08:41:09,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742099_1275 (size=12301) 2024-11-19T08:41:09,485 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/7f7e9497a18c40578d99f7f42907e099 2024-11-19T08:41:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742100_1276 (size=12983) 2024-11-19T08:41:09,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/1480f6b465394a8dafb708891a59b74e is 50, key is test_row_0/C:col10/1732005667847/Put/seqid=0 2024-11-19T08:41:09,496 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/09f09ef7a2774951a8ee697aaca3d1cd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/09f09ef7a2774951a8ee697aaca3d1cd 2024-11-19T08:41:09,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742101_1277 (size=12301) 2024-11-19T08:41:09,502 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/1480f6b465394a8dafb708891a59b74e 2024-11-19T08:41:09,503 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into 09f09ef7a2774951a8ee697aaca3d1cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:09,503 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:09,503 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=13, startTime=1732005669040; duration=0sec 2024-11-19T08:41:09,503 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:09,503 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/2cb8e390680e41cc8fc2ab56c2f7d795 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2cb8e390680e41cc8fc2ab56c2f7d795 2024-11-19T08:41:09,512 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2cb8e390680e41cc8fc2ab56c2f7d795, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T08:41:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/7f7e9497a18c40578d99f7f42907e099 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7f7e9497a18c40578d99f7f42907e099 2024-11-19T08:41:09,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7f7e9497a18c40578d99f7f42907e099, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T08:41:09,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/1480f6b465394a8dafb708891a59b74e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1480f6b465394a8dafb708891a59b74e 2024-11-19T08:41:09,523 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1480f6b465394a8dafb708891a59b74e, entries=150, sequenceid=313, filesize=12.0 K 2024-11-19T08:41:09,524 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 10e67dc5eaf131cd78dc99b8ad946211 in 472ms, sequenceid=313, compaction requested=false 2024-11-19T08:41:09,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:09,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:09,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T08:41:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T08:41:09,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T08:41:09,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6970 sec 2024-11-19T08:41:09,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7010 sec 2024-11-19T08:41:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T08:41:09,986 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:09,987 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-19T08:41:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T08:41:09,989 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:09,990 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:09,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:10,061 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:10,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:41:10,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/2bb6e5bbf7f54364ab6e5108e785c24c is 50, key is test_row_0/A:col10/1732005670060/Put/seqid=0 2024-11-19T08:41:10,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742102_1278 (size=14741) 2024-11-19T08:41:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T08:41:10,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T08:41:10,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:10,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005730161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:10,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005730276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:10,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T08:41:10,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:10,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T08:41:10,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T08:41:10,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:10,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:10,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/2bb6e5bbf7f54364ab6e5108e785c24c 2024-11-19T08:41:10,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8b25b31e780541d0ba6f781eafb82909 is 50, key is test_row_0/B:col10/1732005670060/Put/seqid=0 2024-11-19T08:41:10,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005730486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:10,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742103_1279 (size=12301) 2024-11-19T08:41:10,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8b25b31e780541d0ba6f781eafb82909 2024-11-19T08:41:10,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/939e1fbd41c64a62aa4d56b5fe68f8bd is 50, key is test_row_0/C:col10/1732005670060/Put/seqid=0 2024-11-19T08:41:10,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742104_1280 (size=12301) 2024-11-19T08:41:10,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/939e1fbd41c64a62aa4d56b5fe68f8bd 2024-11-19T08:41:10,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/2bb6e5bbf7f54364ab6e5108e785c24c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2bb6e5bbf7f54364ab6e5108e785c24c 2024-11-19T08:41:10,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2bb6e5bbf7f54364ab6e5108e785c24c, entries=200, sequenceid=327, filesize=14.4 K 2024-11-19T08:41:10,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8b25b31e780541d0ba6f781eafb82909 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8b25b31e780541d0ba6f781eafb82909 2024-11-19T08:41:10,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8b25b31e780541d0ba6f781eafb82909, entries=150, sequenceid=327, filesize=12.0 K 2024-11-19T08:41:10,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/939e1fbd41c64a62aa4d56b5fe68f8bd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/939e1fbd41c64a62aa4d56b5fe68f8bd 2024-11-19T08:41:10,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/939e1fbd41c64a62aa4d56b5fe68f8bd, entries=150, sequenceid=327, filesize=12.0 K 2024-11-19T08:41:10,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 10e67dc5eaf131cd78dc99b8ad946211 in 519ms, sequenceid=327, compaction requested=true 2024-11-19T08:41:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:10,581 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:10,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:10,581 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:10,582 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:10,582 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:10,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:10,582 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,582 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/61358f1c05084309b6550e2f8826779d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7f7e9497a18c40578d99f7f42907e099, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8b25b31e780541d0ba6f781eafb82909] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.7 K 2024-11-19T08:41:10,582 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:10,582 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:10,582 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,582 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a37dbe2b809142349f837e99451614a3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2cb8e390680e41cc8fc2ab56c2f7d795, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2bb6e5bbf7f54364ab6e5108e785c24c] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=39.1 K 2024-11-19T08:41:10,583 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 61358f1c05084309b6550e2f8826779d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732005667775 2024-11-19T08:41:10,583 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a37dbe2b809142349f837e99451614a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732005667775 2024-11-19T08:41:10,583 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f7e9497a18c40578d99f7f42907e099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732005667847 2024-11-19T08:41:10,583 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cb8e390680e41cc8fc2ab56c2f7d795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732005667847 2024-11-19T08:41:10,584 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b25b31e780541d0ba6f781eafb82909, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732005670052 2024-11-19T08:41:10,584 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bb6e5bbf7f54364ab6e5108e785c24c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732005670052 2024-11-19T08:41:10,600 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:10,601 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/7ef8467f49584d229ce9a830a5cf7498 is 50, key is test_row_0/B:col10/1732005670060/Put/seqid=0 2024-11-19T08:41:10,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T08:41:10,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,603 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:41:10,606 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#241 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:10,606 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/f27f691dd57448138c6fcb079e610afa is 50, key is test_row_0/A:col10/1732005670060/Put/seqid=0 2024-11-19T08:41:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T08:41:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/cdf6a18f72c24de28b8a114455842136 is 50, key is test_row_0/A:col10/1732005670151/Put/seqid=0 2024-11-19T08:41:10,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742105_1281 (size=13085) 2024-11-19T08:41:10,644 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/7ef8467f49584d229ce9a830a5cf7498 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7ef8467f49584d229ce9a830a5cf7498 2024-11-19T08:41:10,650 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into 7ef8467f49584d229ce9a830a5cf7498(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:10,650 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:10,650 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=13, startTime=1732005670581; duration=0sec 2024-11-19T08:41:10,651 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:10,651 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:10,651 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:10,653 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:10,653 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:10,653 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:10,654 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/09f09ef7a2774951a8ee697aaca3d1cd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1480f6b465394a8dafb708891a59b74e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/939e1fbd41c64a62aa4d56b5fe68f8bd] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.7 K 2024-11-19T08:41:10,654 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 09f09ef7a2774951a8ee697aaca3d1cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732005667775 2024-11-19T08:41:10,654 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 1480f6b465394a8dafb708891a59b74e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732005667847 2024-11-19T08:41:10,655 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 939e1fbd41c64a62aa4d56b5fe68f8bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732005670052 2024-11-19T08:41:10,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742106_1282 (size=13085) 2024-11-19T08:41:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742107_1283 (size=12301) 2024-11-19T08:41:10,669 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:10,669 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/68b579e27edb4563aa81d07276d3b9d5 is 50, key is test_row_0/C:col10/1732005670060/Put/seqid=0 2024-11-19T08:41:10,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742108_1284 (size=13085) 2024-11-19T08:41:10,679 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/68b579e27edb4563aa81d07276d3b9d5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/68b579e27edb4563aa81d07276d3b9d5 2024-11-19T08:41:10,683 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into 68b579e27edb4563aa81d07276d3b9d5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:10,684 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:10,684 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=13, startTime=1732005670582; duration=0sec 2024-11-19T08:41:10,684 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:10,684 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:10,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005730820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005730926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:11,061 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/cdf6a18f72c24de28b8a114455842136 2024-11-19T08:41:11,062 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/f27f691dd57448138c6fcb079e610afa as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f27f691dd57448138c6fcb079e610afa 2024-11-19T08:41:11,065 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into f27f691dd57448138c6fcb079e610afa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:11,066 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:11,066 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=13, startTime=1732005670581; duration=0sec 2024-11-19T08:41:11,066 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:11,066 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:11,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/afe0d05d80274491a84239608f81ffd5 is 50, key is test_row_0/B:col10/1732005670151/Put/seqid=0 2024-11-19T08:41:11,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742109_1285 (size=12301) 2024-11-19T08:41:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T08:41:11,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005731136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005731446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:11,473 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/afe0d05d80274491a84239608f81ffd5 2024-11-19T08:41:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/0b2ee760efe5400e942e1b43b437d00d is 50, key is test_row_0/C:col10/1732005670151/Put/seqid=0 2024-11-19T08:41:11,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742110_1286 (size=12301) 2024-11-19T08:41:11,499 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/0b2ee760efe5400e942e1b43b437d00d 2024-11-19T08:41:11,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/cdf6a18f72c24de28b8a114455842136 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/cdf6a18f72c24de28b8a114455842136 2024-11-19T08:41:11,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/cdf6a18f72c24de28b8a114455842136, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T08:41:11,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/afe0d05d80274491a84239608f81ffd5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/afe0d05d80274491a84239608f81ffd5 2024-11-19T08:41:11,522 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/afe0d05d80274491a84239608f81ffd5, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T08:41:11,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/0b2ee760efe5400e942e1b43b437d00d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0b2ee760efe5400e942e1b43b437d00d 2024-11-19T08:41:11,527 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0b2ee760efe5400e942e1b43b437d00d, entries=150, sequenceid=352, filesize=12.0 K 2024-11-19T08:41:11,528 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 10e67dc5eaf131cd78dc99b8ad946211 in 925ms, sequenceid=352, compaction requested=false 2024-11-19T08:41:11,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:11,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:11,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-19T08:41:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-19T08:41:11,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T08:41:11,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5400 sec 2024-11-19T08:41:11,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.5450 sec 2024-11-19T08:41:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:11,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:11,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:41:11,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/a9e5960cf6644ab8864d14c0d61f5486 is 50, key is test_row_0/A:col10/1732005670815/Put/seqid=0 2024-11-19T08:41:11,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742111_1287 (size=14741) 2024-11-19T08:41:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005731998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005731998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005731999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005732000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005732005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005732116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005732116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005732116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005732116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005732116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T08:41:12,136 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:12,136 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-19T08:41:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T08:41:12,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:12,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:12,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T08:41:12,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T08:41:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005732326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005732326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005732326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005732327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005732327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/a9e5960cf6644ab8864d14c0d61f5486 2024-11-19T08:41:12,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8440373af6b441b4be6cb2fc630de2f3 is 50, key is test_row_0/B:col10/1732005670815/Put/seqid=0 2024-11-19T08:41:12,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742112_1288 (size=12301) 2024-11-19T08:41:12,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T08:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T08:41:12,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T08:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005732636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005732636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005732636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005732636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005732636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:12,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:12,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T08:41:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T08:41:12,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8440373af6b441b4be6cb2fc630de2f3 2024-11-19T08:41:12,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7b798e956a0c4d41bc55ee7b0cd39575 is 50, key is test_row_0/C:col10/1732005670815/Put/seqid=0 2024-11-19T08:41:12,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742113_1289 (size=12301) 2024-11-19T08:41:12,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T08:41:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:12,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:13,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T08:41:13,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:13,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005733146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005733146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005733147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005733147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005733147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:13,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7b798e956a0c4d41bc55ee7b0cd39575 2024-11-19T08:41:13,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/a9e5960cf6644ab8864d14c0d61f5486 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a9e5960cf6644ab8864d14c0d61f5486 2024-11-19T08:41:13,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a9e5960cf6644ab8864d14c0d61f5486, entries=200, sequenceid=368, filesize=14.4 K 2024-11-19T08:41:13,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/8440373af6b441b4be6cb2fc630de2f3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8440373af6b441b4be6cb2fc630de2f3 2024-11-19T08:41:13,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8440373af6b441b4be6cb2fc630de2f3, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T08:41:13,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7b798e956a0c4d41bc55ee7b0cd39575 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7b798e956a0c4d41bc55ee7b0cd39575 2024-11-19T08:41:13,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7b798e956a0c4d41bc55ee7b0cd39575, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T08:41:13,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 10e67dc5eaf131cd78dc99b8ad946211 in 1256ms, sequenceid=368, compaction requested=true 2024-11-19T08:41:13,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:13,206 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:13,206 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:13,207 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:13,207 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:13,207 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:13,207 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:13,207 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:13,207 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:13,207 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f27f691dd57448138c6fcb079e610afa, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/cdf6a18f72c24de28b8a114455842136, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a9e5960cf6644ab8864d14c0d61f5486] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=39.2 K 2024-11-19T08:41:13,207 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7ef8467f49584d229ce9a830a5cf7498, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/afe0d05d80274491a84239608f81ffd5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8440373af6b441b4be6cb2fc630de2f3] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.8 K 2024-11-19T08:41:13,208 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef8467f49584d229ce9a830a5cf7498, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732005670052 2024-11-19T08:41:13,208 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f27f691dd57448138c6fcb079e610afa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732005670052 2024-11-19T08:41:13,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T08:41:13,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:13,208 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:41:13,209 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf6a18f72c24de28b8a114455842136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732005670151 2024-11-19T08:41:13,209 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting afe0d05d80274491a84239608f81ffd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732005670151 2024-11-19T08:41:13,210 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e5960cf6644ab8864d14c0d61f5486, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732005670815 2024-11-19T08:41:13,210 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8440373af6b441b4be6cb2fc630de2f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732005670815 2024-11-19T08:41:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/e834c8a7b85a43f6b732e9650e93338f is 50, key is test_row_0/A:col10/1732005671973/Put/seqid=0 2024-11-19T08:41:13,224 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:13,224 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/a0bde6e57c0c4d0d97b576091adb315f is 50, key is test_row_0/A:col10/1732005670815/Put/seqid=0 2024-11-19T08:41:13,237 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:13,237 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/a10b7701a54f4bfdaf6de97049957606 is 50, key is test_row_0/B:col10/1732005670815/Put/seqid=0 2024-11-19T08:41:13,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742114_1290 (size=12301) 2024-11-19T08:41:13,245 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/e834c8a7b85a43f6b732e9650e93338f 2024-11-19T08:41:13,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/41efeec47bf14175b8a5528375dadf47 is 50, key is test_row_0/B:col10/1732005671973/Put/seqid=0 2024-11-19T08:41:13,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742116_1292 (size=13187) 2024-11-19T08:41:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T08:41:13,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742115_1291 (size=13187) 2024-11-19T08:41:13,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742117_1293 (size=12301) 2024-11-19T08:41:13,674 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/a10b7701a54f4bfdaf6de97049957606 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/a10b7701a54f4bfdaf6de97049957606 2024-11-19T08:41:13,678 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into a10b7701a54f4bfdaf6de97049957606(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:13,678 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:13,678 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=13, startTime=1732005673206; duration=0sec 2024-11-19T08:41:13,678 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:13,678 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:13,679 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:13,679 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:13,679 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:13,680 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:13,680 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/68b579e27edb4563aa81d07276d3b9d5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0b2ee760efe5400e942e1b43b437d00d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7b798e956a0c4d41bc55ee7b0cd39575] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.8 K 2024-11-19T08:41:13,680 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 68b579e27edb4563aa81d07276d3b9d5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732005670052 2024-11-19T08:41:13,680 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2ee760efe5400e942e1b43b437d00d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732005670151 2024-11-19T08:41:13,681 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b798e956a0c4d41bc55ee7b0cd39575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732005670815 2024-11-19T08:41:13,681 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/41efeec47bf14175b8a5528375dadf47 2024-11-19T08:41:13,683 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/a0bde6e57c0c4d0d97b576091adb315f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a0bde6e57c0c4d0d97b576091adb315f 2024-11-19T08:41:13,688 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:13,688 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into a0bde6e57c0c4d0d97b576091adb315f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:13,688 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:13,688 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=13, startTime=1732005673205; duration=0sec 2024-11-19T08:41:13,689 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/025766a8c79e4533b845f868a29346b6 is 50, key is test_row_0/C:col10/1732005670815/Put/seqid=0 2024-11-19T08:41:13,689 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:13,689 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/1819d3ad5f564ac78f39449653b4b714 is 50, key is test_row_0/C:col10/1732005671973/Put/seqid=0 2024-11-19T08:41:13,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742118_1294 (size=13187) 2024-11-19T08:41:13,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742119_1295 (size=12301) 2024-11-19T08:41:13,695 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/1819d3ad5f564ac78f39449653b4b714 2024-11-19T08:41:13,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/e834c8a7b85a43f6b732e9650e93338f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e834c8a7b85a43f6b732e9650e93338f 2024-11-19T08:41:13,701 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e834c8a7b85a43f6b732e9650e93338f, entries=150, sequenceid=391, filesize=12.0 K 2024-11-19T08:41:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/41efeec47bf14175b8a5528375dadf47 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/41efeec47bf14175b8a5528375dadf47 2024-11-19T08:41:13,708 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/41efeec47bf14175b8a5528375dadf47, entries=150, sequenceid=391, filesize=12.0 K 2024-11-19T08:41:13,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/1819d3ad5f564ac78f39449653b4b714 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1819d3ad5f564ac78f39449653b4b714 2024-11-19T08:41:13,713 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1819d3ad5f564ac78f39449653b4b714, entries=150, sequenceid=391, filesize=12.0 K 2024-11-19T08:41:13,714 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 10e67dc5eaf131cd78dc99b8ad946211 in 506ms, sequenceid=391, compaction requested=false 2024-11-19T08:41:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-19T08:41:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-19T08:41:13,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T08:41:13,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5770 sec 2024-11-19T08:41:13,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.5820 sec 2024-11-19T08:41:14,097 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/025766a8c79e4533b845f868a29346b6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/025766a8c79e4533b845f868a29346b6 2024-11-19T08:41:14,102 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into 025766a8c79e4533b845f868a29346b6(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:14,102 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:14,102 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=13, startTime=1732005673206; duration=0sec 2024-11-19T08:41:14,102 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:14,102 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:14,163 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:14,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:41:14,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/61a2aa9cb775481484624421040002e9 is 50, key is test_row_0/A:col10/1732005674161/Put/seqid=0 2024-11-19T08:41:14,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742120_1296 (size=14741) 2024-11-19T08:41:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005734187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005734189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005734192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005734193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005734193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T08:41:14,296 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:14,297 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-19T08:41:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T08:41:14,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:14,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:14,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:14,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005734296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005734296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005734306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005734306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005734306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T08:41:14,442 INFO [Thread-1014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1014. 2024-11-19T08:41:14,442 DEBUG [Thread-1014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:14,442 DEBUG [Thread-1014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,442 DEBUG [Thread-1014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:14,443 INFO [Thread-1018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1018. 2024-11-19T08:41:14,443 DEBUG [Thread-1018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:14,443 DEBUG [Thread-1018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,443 DEBUG [Thread-1018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:14,444 INFO [Thread-1012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1012. 2024-11-19T08:41:14,444 DEBUG [Thread-1012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:14,444 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,444 DEBUG [Thread-1012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:14,445 INFO [Thread-1010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1010. 2024-11-19T08:41:14,445 DEBUG [Thread-1010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:14,445 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,445 DEBUG [Thread-1010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:14,446 INFO [Thread-1016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1016. 2024-11-19T08:41:14,446 DEBUG [Thread-1016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:14,446 DEBUG [Thread-1016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,446 DEBUG [Thread-1016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:14,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:14,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005734506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005734507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005734516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005734516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005734516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/61a2aa9cb775481484624421040002e9 2024-11-19T08:41:14,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/e067541091cc40768ce5f929e368391f is 50, key is test_row_0/B:col10/1732005674161/Put/seqid=0 2024-11-19T08:41:14,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742121_1297 (size=12301) 2024-11-19T08:41:14,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T08:41:14,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:14,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:14,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005734817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005734817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005734827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005734826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005734827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:14,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:14,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:14,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:14,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T08:41:15,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/e067541091cc40768ce5f929e368391f 2024-11-19T08:41:15,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7e243319e81f4fdaafb62dbb28f6f5ed is 50, key is test_row_0/C:col10/1732005674161/Put/seqid=0 2024-11-19T08:41:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742122_1298 (size=12301) 2024-11-19T08:41:15,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:15,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:15,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58022 deadline: 1732005735326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57956 deadline: 1732005735326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58024 deadline: 1732005735335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732005735336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57954 deadline: 1732005735336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:15,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:15,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:15,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:15,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7e243319e81f4fdaafb62dbb28f6f5ed 2024-11-19T08:41:15,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/61a2aa9cb775481484624421040002e9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/61a2aa9cb775481484624421040002e9 2024-11-19T08:41:15,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/61a2aa9cb775481484624421040002e9, entries=200, sequenceid=405, filesize=14.4 K 2024-11-19T08:41:15,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/e067541091cc40768ce5f929e368391f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/e067541091cc40768ce5f929e368391f 2024-11-19T08:41:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T08:41:15,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/e067541091cc40768ce5f929e368391f, entries=150, sequenceid=405, filesize=12.0 K 2024-11-19T08:41:15,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7e243319e81f4fdaafb62dbb28f6f5ed as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7e243319e81f4fdaafb62dbb28f6f5ed 2024-11-19T08:41:15,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7e243319e81f4fdaafb62dbb28f6f5ed, entries=150, sequenceid=405, filesize=12.0 K 2024-11-19T08:41:15,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 10e67dc5eaf131cd78dc99b8ad946211 in 1282ms, sequenceid=405, compaction requested=true 2024-11-19T08:41:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:15,445 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:15,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:15,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10e67dc5eaf131cd78dc99b8ad946211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:15,446 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:15,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:15,446 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:15,446 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:15,446 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/B is initiating minor compaction (all files) 2024-11-19T08:41:15,446 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/A is initiating minor compaction (all files) 2024-11-19T08:41:15,446 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/B in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,447 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/A in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,447 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/a10b7701a54f4bfdaf6de97049957606, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/41efeec47bf14175b8a5528375dadf47, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/e067541091cc40768ce5f929e368391f] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.9 K 2024-11-19T08:41:15,447 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a0bde6e57c0c4d0d97b576091adb315f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e834c8a7b85a43f6b732e9650e93338f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/61a2aa9cb775481484624421040002e9] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=39.3 K 2024-11-19T08:41:15,447 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a10b7701a54f4bfdaf6de97049957606, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732005670815 2024-11-19T08:41:15,447 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0bde6e57c0c4d0d97b576091adb315f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732005670815 2024-11-19T08:41:15,447 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting e834c8a7b85a43f6b732e9650e93338f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732005671973 2024-11-19T08:41:15,447 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 41efeec47bf14175b8a5528375dadf47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732005671973 2024-11-19T08:41:15,448 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a2aa9cb775481484624421040002e9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732005674156 2024-11-19T08:41:15,448 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting e067541091cc40768ce5f929e368391f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732005674156 2024-11-19T08:41:15,454 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:15,454 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:15,455 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/ef58b4321cb8475d8fa0e003e5f163a6 is 50, key is test_row_0/B:col10/1732005674161/Put/seqid=0 2024-11-19T08:41:15,455 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/f11b4bf5fbc8424fbcd785e34f0c24e8 is 50, key is test_row_0/A:col10/1732005674161/Put/seqid=0 2024-11-19T08:41:15,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742124_1300 (size=13289) 2024-11-19T08:41:15,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742123_1299 (size=13289) 2024-11-19T08:41:15,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T08:41:15,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,546 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:41:15,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/59e7bce947e64a0e8cc70586b2ec9ec3 is 50, key is test_row_0/A:col10/1732005674186/Put/seqid=0 2024-11-19T08:41:15,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742125_1301 (size=12301) 2024-11-19T08:41:15,863 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/f11b4bf5fbc8424fbcd785e34f0c24e8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f11b4bf5fbc8424fbcd785e34f0c24e8 2024-11-19T08:41:15,863 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/ef58b4321cb8475d8fa0e003e5f163a6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ef58b4321cb8475d8fa0e003e5f163a6 2024-11-19T08:41:15,867 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/A of 10e67dc5eaf131cd78dc99b8ad946211 into f11b4bf5fbc8424fbcd785e34f0c24e8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:15,867 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/B of 10e67dc5eaf131cd78dc99b8ad946211 into ef58b4321cb8475d8fa0e003e5f163a6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:15,867 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:15,867 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:15,867 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/A, priority=13, startTime=1732005675445; duration=0sec 2024-11-19T08:41:15,867 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/B, priority=13, startTime=1732005675445; duration=0sec 2024-11-19T08:41:15,867 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:15,867 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:A 2024-11-19T08:41:15,867 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:15,867 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:B 2024-11-19T08:41:15,867 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:15,868 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:15,868 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 10e67dc5eaf131cd78dc99b8ad946211/C is initiating minor compaction (all files) 2024-11-19T08:41:15,868 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10e67dc5eaf131cd78dc99b8ad946211/C in TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:15,868 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/025766a8c79e4533b845f868a29346b6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1819d3ad5f564ac78f39449653b4b714, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7e243319e81f4fdaafb62dbb28f6f5ed] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp, totalSize=36.9 K 2024-11-19T08:41:15,869 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 025766a8c79e4533b845f868a29346b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732005670815 2024-11-19T08:41:15,869 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1819d3ad5f564ac78f39449653b4b714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732005671973 2024-11-19T08:41:15,869 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e243319e81f4fdaafb62dbb28f6f5ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732005674156 2024-11-19T08:41:15,875 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10e67dc5eaf131cd78dc99b8ad946211#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:15,876 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/9e8e22758bef46a4a1244928da5d2fc5 is 50, key is test_row_0/C:col10/1732005674161/Put/seqid=0 2024-11-19T08:41:15,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742126_1302 (size=13289) 2024-11-19T08:41:15,954 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/59e7bce947e64a0e8cc70586b2ec9ec3 2024-11-19T08:41:15,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/dcceb7b870784441a6596e6bc5eb56b8 is 50, key is test_row_0/B:col10/1732005674186/Put/seqid=0 2024-11-19T08:41:15,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742127_1303 (size=12301) 2024-11-19T08:41:16,291 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/9e8e22758bef46a4a1244928da5d2fc5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9e8e22758bef46a4a1244928da5d2fc5 2024-11-19T08:41:16,299 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10e67dc5eaf131cd78dc99b8ad946211/C of 10e67dc5eaf131cd78dc99b8ad946211 into 9e8e22758bef46a4a1244928da5d2fc5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:16,299 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:16,299 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., storeName=10e67dc5eaf131cd78dc99b8ad946211/C, priority=13, startTime=1732005675446; duration=0sec 2024-11-19T08:41:16,300 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:16,300 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10e67dc5eaf131cd78dc99b8ad946211:C 2024-11-19T08:41:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:16,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. as already flushing 2024-11-19T08:41:16,338 INFO [Thread-1003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1003. 2024-11-19T08:41:16,338 DEBUG [Thread-1003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:16,338 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,338 INFO [Thread-1001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1001. 2024-11-19T08:41:16,338 DEBUG [Thread-1001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:16,338 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,338 DEBUG [Thread-1003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:16,338 DEBUG [Thread-1001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:16,348 INFO [Thread-1005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1005. 2024-11-19T08:41:16,348 DEBUG [Thread-1005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:16,348 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,348 DEBUG [Thread-1005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:16,358 INFO [Thread-1007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1007. 2024-11-19T08:41:16,358 INFO [Thread-999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-999. 2024-11-19T08:41:16,358 DEBUG [Thread-1007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:16,358 DEBUG [Thread-999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:16,358 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,358 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,358 DEBUG [Thread-1007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,358 DEBUG [Thread-999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:16,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:16,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:16,364 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/dcceb7b870784441a6596e6bc5eb56b8 2024-11-19T08:41:16,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7cbec3e931d44ef0953a99668a601dcb is 50, key is test_row_0/C:col10/1732005674186/Put/seqid=0 2024-11-19T08:41:16,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742128_1304 (size=12301) 2024-11-19T08:41:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T08:41:16,776 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7cbec3e931d44ef0953a99668a601dcb 2024-11-19T08:41:16,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/59e7bce947e64a0e8cc70586b2ec9ec3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/59e7bce947e64a0e8cc70586b2ec9ec3 2024-11-19T08:41:16,790 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/59e7bce947e64a0e8cc70586b2ec9ec3, entries=150, sequenceid=431, filesize=12.0 K 2024-11-19T08:41:16,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/dcceb7b870784441a6596e6bc5eb56b8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/dcceb7b870784441a6596e6bc5eb56b8 2024-11-19T08:41:16,795 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/dcceb7b870784441a6596e6bc5eb56b8, entries=150, sequenceid=431, filesize=12.0 K 2024-11-19T08:41:16,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/7cbec3e931d44ef0953a99668a601dcb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7cbec3e931d44ef0953a99668a601dcb 2024-11-19T08:41:16,799 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7cbec3e931d44ef0953a99668a601dcb, entries=150, sequenceid=431, filesize=12.0 K 2024-11-19T08:41:16,800 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=33.54 KB/34350 for 10e67dc5eaf131cd78dc99b8ad946211 in 1254ms, sequenceid=431, compaction requested=false 2024-11-19T08:41:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 10e67dc5eaf131cd78dc99b8ad946211: 2024-11-19T08:41:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-19T08:41:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-19T08:41:16,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T08:41:16,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5020 sec 2024-11-19T08:41:16,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.5060 sec 2024-11-19T08:41:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T08:41:18,467 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4192 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4134 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4182 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4184 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4097 2024-11-19T08:41:18,467 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T08:41:18,467 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:41:18,468 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:41:18,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:18,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:18,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:18,469 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T08:41:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T08:41:18,472 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005678472"}]},"ts":"1732005678472"} 2024-11-19T08:41:18,474 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T08:41:18,474 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T08:41:18,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:41:18,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, UNASSIGN}] 2024-11-19T08:41:18,477 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, UNASSIGN 2024-11-19T08:41:18,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=10e67dc5eaf131cd78dc99b8ad946211, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:18,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, UNASSIGN because future has completed 2024-11-19T08:41:18,479 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:41:18,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:41:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T08:41:18,632 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:18,632 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:41:18,632 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 10e67dc5eaf131cd78dc99b8ad946211, disabling compactions & flushes 2024-11-19T08:41:18,632 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:18,632 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:18,632 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. after waiting 0 ms 2024-11-19T08:41:18,632 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:18,632 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 10e67dc5eaf131cd78dc99b8ad946211 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T08:41:18,637 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ec72cf81951b48f089354492accc42cb is 50, key is test_row_0/A:col10/1732005676356/Put/seqid=0 2024-11-19T08:41:18,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742129_1305 (size=12301) 2024-11-19T08:41:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T08:41:19,042 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ec72cf81951b48f089354492accc42cb 2024-11-19T08:41:19,049 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/27dcd311a26948e9b46f2e85042e14aa is 50, key is test_row_0/B:col10/1732005676356/Put/seqid=0 2024-11-19T08:41:19,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742130_1306 (size=12301) 2024-11-19T08:41:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T08:41:19,453 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/27dcd311a26948e9b46f2e85042e14aa 2024-11-19T08:41:19,461 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/c7a87b8788fd4f5e836e44fa33581972 is 50, key is test_row_0/C:col10/1732005676356/Put/seqid=0 2024-11-19T08:41:19,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742131_1307 (size=12301) 2024-11-19T08:41:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T08:41:19,867 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/c7a87b8788fd4f5e836e44fa33581972 2024-11-19T08:41:19,873 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/A/ec72cf81951b48f089354492accc42cb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ec72cf81951b48f089354492accc42cb 2024-11-19T08:41:19,877 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ec72cf81951b48f089354492accc42cb, entries=150, sequenceid=442, filesize=12.0 K 2024-11-19T08:41:19,878 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/B/27dcd311a26948e9b46f2e85042e14aa as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/27dcd311a26948e9b46f2e85042e14aa 2024-11-19T08:41:19,881 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/27dcd311a26948e9b46f2e85042e14aa, entries=150, sequenceid=442, filesize=12.0 K 2024-11-19T08:41:19,882 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/.tmp/C/c7a87b8788fd4f5e836e44fa33581972 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c7a87b8788fd4f5e836e44fa33581972 2024-11-19T08:41:19,884 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c7a87b8788fd4f5e836e44fa33581972, entries=150, sequenceid=442, filesize=12.0 K 2024-11-19T08:41:19,885 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 10e67dc5eaf131cd78dc99b8ad946211 in 1253ms, sequenceid=442, compaction requested=true 2024-11-19T08:41:19,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/95b6282aadee43ca8f597572bb6b6120, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5cfcf2fde5624a22afa246e3cf3db7b6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ef1b15b73ad7462993e3d659e5588b1b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/211a76635417400091b329ffcf72a7dc, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/75fa2eaec2b8449183880444700d0468, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/054f6d45d9684c6eb5dc348aa1e56d92, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ca3c9cb35888411193f416c19a0692f6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e641b5b2e04949a28ea6c157918518fd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d9151ff8820a42b4982a39abb71daaa6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/9e56ffe0370e4f80a0fd48e17aa2253e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5099c0e5b8f74c558d2cfcfdb993077e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/1691ca70a8d84762b9f3f5ca5dad811a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/af2f9b9a4f444101a818a201b23ccab0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/fe58281439404993aa8b7ea36ee2b766, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d64dd338f76c4200a8ef1d4f61dd3ed4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/7f79a2a664824a2e928a50058fdcb9a4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/3ad94a6d14194443926b941b9452bf56, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ae416a0e223c4ff58b709fd8ae8b93cd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/06f3cfa3403543b7a699cae30f46be86, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/0f657865eafb46ddab28db507e51c72d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a37dbe2b809142349f837e99451614a3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2cb8e390680e41cc8fc2ab56c2f7d795, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2bb6e5bbf7f54364ab6e5108e785c24c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f27f691dd57448138c6fcb079e610afa, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/cdf6a18f72c24de28b8a114455842136, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a9e5960cf6644ab8864d14c0d61f5486, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a0bde6e57c0c4d0d97b576091adb315f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e834c8a7b85a43f6b732e9650e93338f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/61a2aa9cb775481484624421040002e9] to archive 2024-11-19T08:41:19,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:19,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/95b6282aadee43ca8f597572bb6b6120 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/95b6282aadee43ca8f597572bb6b6120 2024-11-19T08:41:19,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5cfcf2fde5624a22afa246e3cf3db7b6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5cfcf2fde5624a22afa246e3cf3db7b6 2024-11-19T08:41:19,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ef1b15b73ad7462993e3d659e5588b1b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ef1b15b73ad7462993e3d659e5588b1b 2024-11-19T08:41:19,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/211a76635417400091b329ffcf72a7dc to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/211a76635417400091b329ffcf72a7dc 2024-11-19T08:41:19,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/75fa2eaec2b8449183880444700d0468 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/75fa2eaec2b8449183880444700d0468 2024-11-19T08:41:19,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/054f6d45d9684c6eb5dc348aa1e56d92 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/054f6d45d9684c6eb5dc348aa1e56d92 2024-11-19T08:41:19,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ca3c9cb35888411193f416c19a0692f6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ca3c9cb35888411193f416c19a0692f6 2024-11-19T08:41:19,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e641b5b2e04949a28ea6c157918518fd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e641b5b2e04949a28ea6c157918518fd 2024-11-19T08:41:19,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d9151ff8820a42b4982a39abb71daaa6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d9151ff8820a42b4982a39abb71daaa6 2024-11-19T08:41:19,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/9e56ffe0370e4f80a0fd48e17aa2253e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/9e56ffe0370e4f80a0fd48e17aa2253e 2024-11-19T08:41:19,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5099c0e5b8f74c558d2cfcfdb993077e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/5099c0e5b8f74c558d2cfcfdb993077e 2024-11-19T08:41:19,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/1691ca70a8d84762b9f3f5ca5dad811a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/1691ca70a8d84762b9f3f5ca5dad811a 2024-11-19T08:41:19,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/af2f9b9a4f444101a818a201b23ccab0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/af2f9b9a4f444101a818a201b23ccab0 2024-11-19T08:41:19,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/fe58281439404993aa8b7ea36ee2b766 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/fe58281439404993aa8b7ea36ee2b766 2024-11-19T08:41:19,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d64dd338f76c4200a8ef1d4f61dd3ed4 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/d64dd338f76c4200a8ef1d4f61dd3ed4 2024-11-19T08:41:19,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/7f79a2a664824a2e928a50058fdcb9a4 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/7f79a2a664824a2e928a50058fdcb9a4 2024-11-19T08:41:19,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/3ad94a6d14194443926b941b9452bf56 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/3ad94a6d14194443926b941b9452bf56 2024-11-19T08:41:19,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ae416a0e223c4ff58b709fd8ae8b93cd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ae416a0e223c4ff58b709fd8ae8b93cd 2024-11-19T08:41:19,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/06f3cfa3403543b7a699cae30f46be86 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/06f3cfa3403543b7a699cae30f46be86 2024-11-19T08:41:19,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/0f657865eafb46ddab28db507e51c72d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/0f657865eafb46ddab28db507e51c72d 2024-11-19T08:41:19,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a37dbe2b809142349f837e99451614a3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a37dbe2b809142349f837e99451614a3 2024-11-19T08:41:19,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2cb8e390680e41cc8fc2ab56c2f7d795 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2cb8e390680e41cc8fc2ab56c2f7d795 2024-11-19T08:41:19,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2bb6e5bbf7f54364ab6e5108e785c24c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/2bb6e5bbf7f54364ab6e5108e785c24c 2024-11-19T08:41:19,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f27f691dd57448138c6fcb079e610afa to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f27f691dd57448138c6fcb079e610afa 2024-11-19T08:41:19,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/cdf6a18f72c24de28b8a114455842136 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/cdf6a18f72c24de28b8a114455842136 2024-11-19T08:41:19,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a9e5960cf6644ab8864d14c0d61f5486 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a9e5960cf6644ab8864d14c0d61f5486 2024-11-19T08:41:19,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a0bde6e57c0c4d0d97b576091adb315f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/a0bde6e57c0c4d0d97b576091adb315f 2024-11-19T08:41:19,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e834c8a7b85a43f6b732e9650e93338f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/e834c8a7b85a43f6b732e9650e93338f 2024-11-19T08:41:19,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/61a2aa9cb775481484624421040002e9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/61a2aa9cb775481484624421040002e9 2024-11-19T08:41:19,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/32f0aed39fc84192842d06f5a21c53cf, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/02eec845682d488a86359a06d8e04d33, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/5f4d3d08a7b446479a94632688f00841, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8828c4b588204bd0b769fc9656c2dd02, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6aa5caabe0e84724b02082bab45aa743, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/d99c3cb17c85456ca69be4b33a54e9ab, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ba8dd83bc34e466fb24ba8fb07e8aeba, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ff9af08370414e05b0e1c1e750f63123, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6e2b26febfc64c0aa74e9592d62354d8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/70a4184607c0489996c626e8eb77361a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/fceb1d8764d44a6aab2903151e0f01dd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8486fec2829541b5bc86907534164efc, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/28271a6783f9431698010f56a673da99, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/011098714c8146c2acfbc3e78b66b3dd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/91826b13e873453ebcc5796ddad68f2e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b7d3b2573cfb49acb6196f6f00b76d8d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/63a80568e8044e92b979c4f77dd5d7bc, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/2999391c738944fe92c7788a1fc56970, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/150e6df3b91f452a9d50e1955b749068, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/61358f1c05084309b6550e2f8826779d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b3e8ca4ce20345f894287bbc6b07960d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7f7e9497a18c40578d99f7f42907e099, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7ef8467f49584d229ce9a830a5cf7498, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8b25b31e780541d0ba6f781eafb82909, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/afe0d05d80274491a84239608f81ffd5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/a10b7701a54f4bfdaf6de97049957606, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8440373af6b441b4be6cb2fc630de2f3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/41efeec47bf14175b8a5528375dadf47, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/e067541091cc40768ce5f929e368391f] to archive 2024-11-19T08:41:19,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:19,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/32f0aed39fc84192842d06f5a21c53cf to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/32f0aed39fc84192842d06f5a21c53cf 2024-11-19T08:41:19,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/02eec845682d488a86359a06d8e04d33 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/02eec845682d488a86359a06d8e04d33 2024-11-19T08:41:19,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/5f4d3d08a7b446479a94632688f00841 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/5f4d3d08a7b446479a94632688f00841 2024-11-19T08:41:19,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8828c4b588204bd0b769fc9656c2dd02 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8828c4b588204bd0b769fc9656c2dd02 2024-11-19T08:41:19,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6aa5caabe0e84724b02082bab45aa743 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6aa5caabe0e84724b02082bab45aa743 2024-11-19T08:41:19,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/d99c3cb17c85456ca69be4b33a54e9ab to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/d99c3cb17c85456ca69be4b33a54e9ab 2024-11-19T08:41:19,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ba8dd83bc34e466fb24ba8fb07e8aeba to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ba8dd83bc34e466fb24ba8fb07e8aeba 2024-11-19T08:41:19,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ff9af08370414e05b0e1c1e750f63123 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ff9af08370414e05b0e1c1e750f63123 2024-11-19T08:41:19,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6e2b26febfc64c0aa74e9592d62354d8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/6e2b26febfc64c0aa74e9592d62354d8 2024-11-19T08:41:19,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/70a4184607c0489996c626e8eb77361a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/70a4184607c0489996c626e8eb77361a 2024-11-19T08:41:19,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/fceb1d8764d44a6aab2903151e0f01dd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/fceb1d8764d44a6aab2903151e0f01dd 2024-11-19T08:41:19,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8486fec2829541b5bc86907534164efc to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8486fec2829541b5bc86907534164efc 2024-11-19T08:41:19,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/28271a6783f9431698010f56a673da99 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/28271a6783f9431698010f56a673da99 2024-11-19T08:41:19,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/011098714c8146c2acfbc3e78b66b3dd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/011098714c8146c2acfbc3e78b66b3dd 2024-11-19T08:41:19,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/91826b13e873453ebcc5796ddad68f2e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/91826b13e873453ebcc5796ddad68f2e 2024-11-19T08:41:19,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b7d3b2573cfb49acb6196f6f00b76d8d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b7d3b2573cfb49acb6196f6f00b76d8d 2024-11-19T08:41:19,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/63a80568e8044e92b979c4f77dd5d7bc to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/63a80568e8044e92b979c4f77dd5d7bc 2024-11-19T08:41:19,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/2999391c738944fe92c7788a1fc56970 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/2999391c738944fe92c7788a1fc56970 2024-11-19T08:41:19,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/150e6df3b91f452a9d50e1955b749068 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/150e6df3b91f452a9d50e1955b749068 2024-11-19T08:41:19,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/61358f1c05084309b6550e2f8826779d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/61358f1c05084309b6550e2f8826779d 2024-11-19T08:41:19,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b3e8ca4ce20345f894287bbc6b07960d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/b3e8ca4ce20345f894287bbc6b07960d 2024-11-19T08:41:19,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7f7e9497a18c40578d99f7f42907e099 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7f7e9497a18c40578d99f7f42907e099 2024-11-19T08:41:19,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7ef8467f49584d229ce9a830a5cf7498 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/7ef8467f49584d229ce9a830a5cf7498 2024-11-19T08:41:19,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8b25b31e780541d0ba6f781eafb82909 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8b25b31e780541d0ba6f781eafb82909 2024-11-19T08:41:19,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/afe0d05d80274491a84239608f81ffd5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/afe0d05d80274491a84239608f81ffd5 2024-11-19T08:41:19,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/a10b7701a54f4bfdaf6de97049957606 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/a10b7701a54f4bfdaf6de97049957606 2024-11-19T08:41:19,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8440373af6b441b4be6cb2fc630de2f3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/8440373af6b441b4be6cb2fc630de2f3 2024-11-19T08:41:19,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/41efeec47bf14175b8a5528375dadf47 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/41efeec47bf14175b8a5528375dadf47 2024-11-19T08:41:19,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/e067541091cc40768ce5f929e368391f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/e067541091cc40768ce5f929e368391f 2024-11-19T08:41:19,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/6ea85ddd0e19499d809a941777748495, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/946f73f7dde44ad099e2b8d3ab069147, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0d4d9e7a1c0d470db9e37bdb23443018, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/501ef0a32f6f4272ad7414543a8c4d4d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b640532053f94beeb6962920ff409d08, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f0b93feb6f1f4be09ec6e28852a5563a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b256044dd0524a76b53a9a47a859af7d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f8edac0d4500444db062281ca2db94e5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/349b45aebe2b4770a0dde9a007ec9195, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/086c1e7e7df045af9b7b2a80060dae7b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/ba23da8089354f2d9b6c139a69b59df9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9a689f00963841a39949ef5a04f16535, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/4310c3307e9e4208b5bbd4ae2749f5a2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c2c9d2912e0c457ca619117ee24af431, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/618cfbef767248b4a89821267a203313, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/d4ac2509c6994d119ceda731b1c71a93, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/39921a3e75c84b859f3045b31c1cf612, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/a2d8b815f57841dd9cd945a5d2f08104, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/2c610d5dbec341d49f73d206ab958105, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/09f09ef7a2774951a8ee697aaca3d1cd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/927ee11d909f4a5e97aaaac8b07d1a03, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1480f6b465394a8dafb708891a59b74e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/68b579e27edb4563aa81d07276d3b9d5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/939e1fbd41c64a62aa4d56b5fe68f8bd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0b2ee760efe5400e942e1b43b437d00d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/025766a8c79e4533b845f868a29346b6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7b798e956a0c4d41bc55ee7b0cd39575, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1819d3ad5f564ac78f39449653b4b714, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7e243319e81f4fdaafb62dbb28f6f5ed] to archive 2024-11-19T08:41:19,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:19,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/6ea85ddd0e19499d809a941777748495 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/6ea85ddd0e19499d809a941777748495 2024-11-19T08:41:19,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/946f73f7dde44ad099e2b8d3ab069147 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/946f73f7dde44ad099e2b8d3ab069147 2024-11-19T08:41:19,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0d4d9e7a1c0d470db9e37bdb23443018 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0d4d9e7a1c0d470db9e37bdb23443018 2024-11-19T08:41:19,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/501ef0a32f6f4272ad7414543a8c4d4d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/501ef0a32f6f4272ad7414543a8c4d4d 2024-11-19T08:41:19,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b640532053f94beeb6962920ff409d08 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b640532053f94beeb6962920ff409d08 2024-11-19T08:41:19,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f0b93feb6f1f4be09ec6e28852a5563a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f0b93feb6f1f4be09ec6e28852a5563a 2024-11-19T08:41:19,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b256044dd0524a76b53a9a47a859af7d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/b256044dd0524a76b53a9a47a859af7d 2024-11-19T08:41:19,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f8edac0d4500444db062281ca2db94e5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/f8edac0d4500444db062281ca2db94e5 2024-11-19T08:41:19,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/349b45aebe2b4770a0dde9a007ec9195 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/349b45aebe2b4770a0dde9a007ec9195 2024-11-19T08:41:19,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/086c1e7e7df045af9b7b2a80060dae7b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/086c1e7e7df045af9b7b2a80060dae7b 2024-11-19T08:41:19,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/ba23da8089354f2d9b6c139a69b59df9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/ba23da8089354f2d9b6c139a69b59df9 2024-11-19T08:41:19,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9a689f00963841a39949ef5a04f16535 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9a689f00963841a39949ef5a04f16535 2024-11-19T08:41:19,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/4310c3307e9e4208b5bbd4ae2749f5a2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/4310c3307e9e4208b5bbd4ae2749f5a2 2024-11-19T08:41:19,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c2c9d2912e0c457ca619117ee24af431 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c2c9d2912e0c457ca619117ee24af431 2024-11-19T08:41:19,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/618cfbef767248b4a89821267a203313 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/618cfbef767248b4a89821267a203313 2024-11-19T08:41:19,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/d4ac2509c6994d119ceda731b1c71a93 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/d4ac2509c6994d119ceda731b1c71a93 2024-11-19T08:41:19,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/39921a3e75c84b859f3045b31c1cf612 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/39921a3e75c84b859f3045b31c1cf612 2024-11-19T08:41:19,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/a2d8b815f57841dd9cd945a5d2f08104 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/a2d8b815f57841dd9cd945a5d2f08104 2024-11-19T08:41:19,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/2c610d5dbec341d49f73d206ab958105 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/2c610d5dbec341d49f73d206ab958105 2024-11-19T08:41:19,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/09f09ef7a2774951a8ee697aaca3d1cd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/09f09ef7a2774951a8ee697aaca3d1cd 2024-11-19T08:41:19,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/927ee11d909f4a5e97aaaac8b07d1a03 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/927ee11d909f4a5e97aaaac8b07d1a03 2024-11-19T08:41:19,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1480f6b465394a8dafb708891a59b74e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1480f6b465394a8dafb708891a59b74e 2024-11-19T08:41:19,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/68b579e27edb4563aa81d07276d3b9d5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/68b579e27edb4563aa81d07276d3b9d5 2024-11-19T08:41:19,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/939e1fbd41c64a62aa4d56b5fe68f8bd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/939e1fbd41c64a62aa4d56b5fe68f8bd 2024-11-19T08:41:19,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0b2ee760efe5400e942e1b43b437d00d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/0b2ee760efe5400e942e1b43b437d00d 2024-11-19T08:41:19,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/025766a8c79e4533b845f868a29346b6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/025766a8c79e4533b845f868a29346b6 2024-11-19T08:41:19,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7b798e956a0c4d41bc55ee7b0cd39575 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7b798e956a0c4d41bc55ee7b0cd39575 2024-11-19T08:41:19,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1819d3ad5f564ac78f39449653b4b714 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/1819d3ad5f564ac78f39449653b4b714 2024-11-19T08:41:19,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7e243319e81f4fdaafb62dbb28f6f5ed to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7e243319e81f4fdaafb62dbb28f6f5ed 2024-11-19T08:41:19,971 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=1 2024-11-19T08:41:19,972 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. 2024-11-19T08:41:19,972 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 10e67dc5eaf131cd78dc99b8ad946211: Waiting for close lock at 1732005678632Running coprocessor pre-close hooks at 1732005678632Disabling compacts and flushes for region at 1732005678632Disabling writes for close at 1732005678632Obtaining lock to block concurrent updates at 1732005678632Preparing flush snapshotting stores in 10e67dc5eaf131cd78dc99b8ad946211 at 1732005678632Finished memstore snapshotting TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732005678633 (+1 ms)Flushing stores of TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211. at 1732005678633Flushing 10e67dc5eaf131cd78dc99b8ad946211/A: creating writer at 1732005678633Flushing 10e67dc5eaf131cd78dc99b8ad946211/A: appending metadata at 1732005678637 (+4 ms)Flushing 10e67dc5eaf131cd78dc99b8ad946211/A: closing flushed file at 1732005678637Flushing 10e67dc5eaf131cd78dc99b8ad946211/B: creating writer at 1732005679046 (+409 ms)Flushing 10e67dc5eaf131cd78dc99b8ad946211/B: appending metadata at 1732005679049 (+3 ms)Flushing 10e67dc5eaf131cd78dc99b8ad946211/B: closing flushed file at 1732005679049Flushing 10e67dc5eaf131cd78dc99b8ad946211/C: creating writer at 1732005679457 (+408 ms)Flushing 10e67dc5eaf131cd78dc99b8ad946211/C: appending metadata at 1732005679460 (+3 ms)Flushing 10e67dc5eaf131cd78dc99b8ad946211/C: closing flushed file at 1732005679460Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ea12910: reopening flushed file at 1732005679872 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@614d1f8f: reopening flushed file at 1732005679877 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d5e9734: reopening flushed file at 1732005679881 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 10e67dc5eaf131cd78dc99b8ad946211 in 1253ms, sequenceid=442, compaction requested=true at 1732005679885 (+4 ms)Writing region close event to WAL at 1732005679968 (+83 ms)Running coprocessor post-close hooks at 1732005679972 (+4 ms)Closed at 1732005679972 2024-11-19T08:41:19,974 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:19,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=10e67dc5eaf131cd78dc99b8ad946211, regionState=CLOSED 2024-11-19T08:41:19,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:41:19,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T08:41:19,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 10e67dc5eaf131cd78dc99b8ad946211, server=3ab37fa97a98,34165,1732005593269 in 1.4970 sec 2024-11-19T08:41:19,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-19T08:41:19,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10e67dc5eaf131cd78dc99b8ad946211, UNASSIGN in 1.5020 sec 2024-11-19T08:41:19,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T08:41:19,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5050 sec 2024-11-19T08:41:19,983 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005679983"}]},"ts":"1732005679983"} 2024-11-19T08:41:19,985 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T08:41:19,985 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T08:41:19,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5170 sec 2024-11-19T08:41:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T08:41:20,617 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:20,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T08:41:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:20,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T08:41:20,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:20,622 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:20,624 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/recovered.edits] 2024-11-19T08:41:20,626 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/59e7bce947e64a0e8cc70586b2ec9ec3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/59e7bce947e64a0e8cc70586b2ec9ec3 2024-11-19T08:41:20,627 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ec72cf81951b48f089354492accc42cb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/ec72cf81951b48f089354492accc42cb 2024-11-19T08:41:20,628 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f11b4bf5fbc8424fbcd785e34f0c24e8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/A/f11b4bf5fbc8424fbcd785e34f0c24e8 2024-11-19T08:41:20,631 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/27dcd311a26948e9b46f2e85042e14aa to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/27dcd311a26948e9b46f2e85042e14aa 2024-11-19T08:41:20,632 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/dcceb7b870784441a6596e6bc5eb56b8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/dcceb7b870784441a6596e6bc5eb56b8 2024-11-19T08:41:20,633 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ef58b4321cb8475d8fa0e003e5f163a6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/B/ef58b4321cb8475d8fa0e003e5f163a6 2024-11-19T08:41:20,635 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7cbec3e931d44ef0953a99668a601dcb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/7cbec3e931d44ef0953a99668a601dcb 2024-11-19T08:41:20,636 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9e8e22758bef46a4a1244928da5d2fc5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/9e8e22758bef46a4a1244928da5d2fc5 2024-11-19T08:41:20,637 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c7a87b8788fd4f5e836e44fa33581972 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/C/c7a87b8788fd4f5e836e44fa33581972 2024-11-19T08:41:20,639 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/recovered.edits/445.seqid to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211/recovered.edits/445.seqid 2024-11-19T08:41:20,640 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/10e67dc5eaf131cd78dc99b8ad946211 2024-11-19T08:41:20,640 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T08:41:20,641 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:20,646 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T08:41:20,648 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T08:41:20,649 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:20,649 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T08:41:20,649 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732005680649"}]},"ts":"9223372036854775807"} 2024-11-19T08:41:20,652 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T08:41:20,652 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 10e67dc5eaf131cd78dc99b8ad946211, NAME => 'TestAcidGuarantees,,1732005652226.10e67dc5eaf131cd78dc99b8ad946211.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T08:41:20,652 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T08:41:20,652 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732005680652"}]},"ts":"9223372036854775807"} 2024-11-19T08:41:20,654 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T08:41:20,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:20,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-19T08:41:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T08:41:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T08:41:20,729 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:20,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 241), OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 482), ProcessCount=11 (was 11), AvailableMemoryMB=6182 (was 5443) - AvailableMemoryMB LEAK? - 2024-11-19T08:41:20,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=6180 2024-11-19T08:41:20,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:41:20,747 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:41:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:20,749 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T08:41:20,749 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:20,749 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-19T08:41:20,750 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T08:41:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T08:41:20,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742132_1308 (size=959) 2024-11-19T08:41:20,757 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:41:20,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742133_1309 (size=53) 2024-11-19T08:41:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T08:41:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T08:41:21,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:41:21,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8f3ff379c750bd3082218a4d8d91a02f, disabling compactions & flushes 2024-11-19T08:41:21,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:21,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:21,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. after waiting 0 ms 2024-11-19T08:41:21,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:21,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:21,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f3ff379c750bd3082218a4d8d91a02f: Waiting for close lock at 1732005681167Disabling compacts and flushes for region at 1732005681167Disabling writes for close at 1732005681167Writing region close event to WAL at 1732005681167Closed at 1732005681167 2024-11-19T08:41:21,170 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T08:41:21,171 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732005681170"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732005681170"}]},"ts":"1732005681170"} 2024-11-19T08:41:21,174 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T08:41:21,175 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T08:41:21,176 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005681175"}]},"ts":"1732005681175"} 2024-11-19T08:41:21,178 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T08:41:21,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, ASSIGN}] 2024-11-19T08:41:21,180 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, ASSIGN 2024-11-19T08:41:21,181 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, ASSIGN; state=OFFLINE, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=false 2024-11-19T08:41:21,332 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:21,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, ASSIGN because future has completed 2024-11-19T08:41:21,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:41:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T08:41:21,489 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:21,489 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:41:21,489 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,489 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:41:21,490 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,490 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,491 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,492 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3ff379c750bd3082218a4d8d91a02f columnFamilyName A 2024-11-19T08:41:21,492 DEBUG [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:21,493 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(327): Store=8f3ff379c750bd3082218a4d8d91a02f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:21,493 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,494 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3ff379c750bd3082218a4d8d91a02f columnFamilyName B 2024-11-19T08:41:21,494 DEBUG [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:21,494 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(327): Store=8f3ff379c750bd3082218a4d8d91a02f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:21,494 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,495 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3ff379c750bd3082218a4d8d91a02f columnFamilyName C 2024-11-19T08:41:21,495 DEBUG [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:21,496 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(327): Store=8f3ff379c750bd3082218a4d8d91a02f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:21,496 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,497 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,497 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,498 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,498 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,498 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:41:21,500 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,502 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:41:21,502 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 8f3ff379c750bd3082218a4d8d91a02f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66363516, jitterRate=-0.011106550693511963}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:41:21,502 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:21,503 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 8f3ff379c750bd3082218a4d8d91a02f: Running coprocessor pre-open hook at 1732005681490Writing region info on filesystem at 1732005681490Initializing all the Stores at 1732005681490Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005681491 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005681491Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005681491Cleaning up temporary data from old regions at 1732005681498 (+7 ms)Running coprocessor post-open hooks at 1732005681502 (+4 ms)Region opened successfully at 1732005681503 (+1 ms) 2024-11-19T08:41:21,503 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., pid=95, masterSystemTime=1732005681485 2024-11-19T08:41:21,505 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:21,505 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:21,506 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=OPEN, openSeqNum=2, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:21,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:41:21,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-19T08:41:21,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 in 174 msec 2024-11-19T08:41:21,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T08:41:21,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, ASSIGN in 331 msec 2024-11-19T08:41:21,512 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T08:41:21,512 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005681512"}]},"ts":"1732005681512"} 2024-11-19T08:41:21,514 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T08:41:21,515 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T08:41:21,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 768 msec 2024-11-19T08:41:21,524 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T08:41:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T08:41:21,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:21,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:21,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:21,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:21,889 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4750c40a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:21,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f27d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:21,893 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:21,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:21,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T08:41:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ab37fa97a98,36325,1732005592414 2024-11-19T08:41:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1804ba5e 2024-11-19T08:41:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T08:41:21,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T08:41:21,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:41:21,900 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:41:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:21,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742134_1310 (size=995) 2024-11-19T08:41:22,313 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T08:41:22,313 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T08:41:22,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:41:22,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, REOPEN/MOVE}] 2024-11-19T08:41:22,318 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, REOPEN/MOVE 2024-11-19T08:41:22,318 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, REOPEN/MOVE because future has completed 2024-11-19T08:41:22,320 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:41:22,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:41:22,472 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,472 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:41:22,472 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 8f3ff379c750bd3082218a4d8d91a02f, disabling compactions & flushes 2024-11-19T08:41:22,472 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:22,472 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:22,472 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. after waiting 0 ms 2024-11-19T08:41:22,472 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:22,476 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T08:41:22,477 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:22,477 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 8f3ff379c750bd3082218a4d8d91a02f: Waiting for close lock at 1732005682472Running coprocessor pre-close hooks at 1732005682472Disabling compacts and flushes for region at 1732005682472Disabling writes for close at 1732005682472Writing region close event to WAL at 1732005682472Running coprocessor post-close hooks at 1732005682476 (+4 ms)Closed at 1732005682476 2024-11-19T08:41:22,477 WARN [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 8f3ff379c750bd3082218a4d8d91a02f to self. 2024-11-19T08:41:22,478 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,479 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=CLOSED 2024-11-19T08:41:22,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:41:22,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-19T08:41:22,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 in 161 msec 2024-11-19T08:41:22,483 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, REOPEN/MOVE; state=CLOSED, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=true 2024-11-19T08:41:22,634 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, REOPEN/MOVE because future has completed 2024-11-19T08:41:22,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:41:22,791 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:22,791 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:41:22,791 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,791 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:41:22,791 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,791 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,792 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,793 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3ff379c750bd3082218a4d8d91a02f columnFamilyName A 2024-11-19T08:41:22,794 DEBUG [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:22,795 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(327): Store=8f3ff379c750bd3082218a4d8d91a02f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:22,795 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,796 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3ff379c750bd3082218a4d8d91a02f columnFamilyName B 2024-11-19T08:41:22,796 DEBUG [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:22,796 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(327): Store=8f3ff379c750bd3082218a4d8d91a02f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:22,796 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,797 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3ff379c750bd3082218a4d8d91a02f columnFamilyName C 2024-11-19T08:41:22,797 DEBUG [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:22,798 INFO [StoreOpener-8f3ff379c750bd3082218a4d8d91a02f-1 {}] regionserver.HStore(327): Store=8f3ff379c750bd3082218a4d8d91a02f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:22,798 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,799 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,800 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,801 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,801 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,801 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:41:22,803 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,804 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 8f3ff379c750bd3082218a4d8d91a02f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70014612, jitterRate=0.04329901933670044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:41:22,804 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,805 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 8f3ff379c750bd3082218a4d8d91a02f: Running coprocessor pre-open hook at 1732005682791Writing region info on filesystem at 1732005682791Initializing all the Stores at 1732005682792 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005682792Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005682792Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005682792Cleaning up temporary data from old regions at 1732005682801 (+9 ms)Running coprocessor post-open hooks at 1732005682804 (+3 ms)Region opened successfully at 1732005682805 (+1 ms) 2024-11-19T08:41:22,806 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., pid=100, masterSystemTime=1732005682787 2024-11-19T08:41:22,808 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:22,808 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:22,808 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=OPEN, openSeqNum=5, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:41:22,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-19T08:41:22,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 in 174 msec 2024-11-19T08:41:22,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T08:41:22,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, REOPEN/MOVE in 494 msec 2024-11-19T08:41:22,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-19T08:41:22,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-19T08:41:22,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-19T08:41:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-19T08:41:22,818 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:22,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,819 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240cebca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,821 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337dcb18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,823 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37288652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,825 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41850861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,827 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11957455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,829 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd12ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,830 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e564dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,832 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73194ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,834 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6481784f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,836 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b48882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e76ac8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520985fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,843 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784b99cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,844 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b888365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,846 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653bb0ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cdc21cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,849 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480c00de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,851 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbe2691, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:22,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:22,853 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da99270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:22,854 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:22,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a21bb41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:22,856 DEBUG [Thread-1431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,857 DEBUG [Thread-1442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,857 DEBUG [Thread-1436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,857 DEBUG [Thread-1434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,857 DEBUG [Thread-1423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,858 DEBUG [Thread-1440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,871 DEBUG [Thread-1438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,876 DEBUG [Thread-1429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:22,882 DEBUG [Thread-1425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T08:41:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,883 DEBUG [Thread-1427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T08:41:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:22,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:22,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:22,885 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:22,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:41:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:22,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:41:22,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7d93f54a24f4f6cb6d68d26232ff655_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005682866/Put/seqid=0 2024-11-19T08:41:22,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742135_1311 (size=12154) 2024-11-19T08:41:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005742959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:22,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005742960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005742960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005742964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:22,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005742971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:22,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T08:41:23,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005743076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005743086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005743086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005743086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005743087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:23,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T08:41:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005743286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005743297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005743297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005743297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005743297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,353 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:23,359 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e7d93f54a24f4f6cb6d68d26232ff655_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7d93f54a24f4f6cb6d68d26232ff655_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:23,360 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/83875ee1b05d4313b6dcdc784698dc1b, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:23,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/83875ee1b05d4313b6dcdc784698dc1b is 175, key is test_row_0/A:col10/1732005682866/Put/seqid=0 2024-11-19T08:41:23,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742136_1312 (size=30955) 2024-11-19T08:41:23,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T08:41:23,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005743596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005743605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005743606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005743606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005743606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:23,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/83875ee1b05d4313b6dcdc784698dc1b 2024-11-19T08:41:23,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/98c9994c8d754046bdf4ed8f1f718b9b is 50, key is test_row_0/B:col10/1732005682866/Put/seqid=0 2024-11-19T08:41:23,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742137_1313 (size=12001) 2024-11-19T08:41:23,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:23,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:23,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:23,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T08:41:24,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005744106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:24,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:24,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:24,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:24,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:24,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:24,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:24,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005744116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:24,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005744117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:24,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005744117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:24,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005744117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:24,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/98c9994c8d754046bdf4ed8f1f718b9b 2024-11-19T08:41:24,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/b87d2ea737074a67bd3a47c298241734 is 50, key is test_row_0/C:col10/1732005682866/Put/seqid=0 2024-11-19T08:41:24,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742138_1314 (size=12001) 2024-11-19T08:41:24,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/b87d2ea737074a67bd3a47c298241734 2024-11-19T08:41:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/83875ee1b05d4313b6dcdc784698dc1b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/83875ee1b05d4313b6dcdc784698dc1b 2024-11-19T08:41:24,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/83875ee1b05d4313b6dcdc784698dc1b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-19T08:41:24,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/98c9994c8d754046bdf4ed8f1f718b9b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/98c9994c8d754046bdf4ed8f1f718b9b 2024-11-19T08:41:24,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/98c9994c8d754046bdf4ed8f1f718b9b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T08:41:24,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/b87d2ea737074a67bd3a47c298241734 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b87d2ea737074a67bd3a47c298241734 2024-11-19T08:41:24,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b87d2ea737074a67bd3a47c298241734, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T08:41:24,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8f3ff379c750bd3082218a4d8d91a02f in 1356ms, sequenceid=16, compaction requested=false 2024-11-19T08:41:24,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:24,263 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T08:41:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:24,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:24,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:24,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T08:41:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:24,416 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T08:41:24,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119717ed0250b984d6cba985115c275524b_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005682960/Put/seqid=0 2024-11-19T08:41:24,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742139_1315 (size=12154) 2024-11-19T08:41:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,448 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119717ed0250b984d6cba985115c275524b_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119717ed0250b984d6cba985115c275524b_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:24,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/0273427c945849748edc154067de328f, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:24,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/0273427c945849748edc154067de328f is 175, key is test_row_0/A:col10/1732005682960/Put/seqid=0 2024-11-19T08:41:24,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742140_1316 (size=30955) 2024-11-19T08:41:24,469 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/0273427c945849748edc154067de328f 2024-11-19T08:41:24,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/749fadc57aa34916a06f9a149ac10375 is 50, key is test_row_0/B:col10/1732005682960/Put/seqid=0 2024-11-19T08:41:24,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742141_1317 (size=12001) 2024-11-19T08:41:24,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/749fadc57aa34916a06f9a149ac10375 2024-11-19T08:41:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/f2dff8a289b847f6af1ee51d89ea69f0 is 50, key is test_row_0/C:col10/1732005682960/Put/seqid=0 2024-11-19T08:41:24,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742142_1318 (size=12001) 2024-11-19T08:41:24,590 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/f2dff8a289b847f6af1ee51d89ea69f0 2024-11-19T08:41:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/0273427c945849748edc154067de328f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/0273427c945849748edc154067de328f 2024-11-19T08:41:24,605 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/0273427c945849748edc154067de328f, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T08:41:24,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/749fadc57aa34916a06f9a149ac10375 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/749fadc57aa34916a06f9a149ac10375 2024-11-19T08:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/749fadc57aa34916a06f9a149ac10375, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T08:41:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/f2dff8a289b847f6af1ee51d89ea69f0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f2dff8a289b847f6af1ee51d89ea69f0 2024-11-19T08:41:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,620 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f2dff8a289b847f6af1ee51d89ea69f0, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T08:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,621 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 8f3ff379c750bd3082218a4d8d91a02f in 204ms, sequenceid=40, compaction requested=false 2024-11-19T08:41:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T08:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T08:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T08:41:24,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7370 sec 2024-11-19T08:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.7440 sec 2024-11-19T08:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T08:41:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T08:41:25,030 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T08:41:25,031 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:25,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T08:41:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T08:41:25,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,185 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-19T08:41:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b683500fdb5e408ab25cd130c064cea4_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_1/A:col10/1732005685179/Put/seqid=0 2024-11-19T08:41:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742143_1319 (size=9714) 2024-11-19T08:41:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,246 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b683500fdb5e408ab25cd130c064cea4_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b683500fdb5e408ab25cd130c064cea4_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/63f88188936f40afac3f703bf3af78fd, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/63f88188936f40afac3f703bf3af78fd is 175, key is test_row_1/A:col10/1732005685179/Put/seqid=0 2024-11-19T08:41:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742144_1320 (size=22361) 2024-11-19T08:41:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:25,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005745310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005745313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005745320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005745322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005745323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T08:41:25,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005745426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005745426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005745436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005745436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005745446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,504 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T08:41:25,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005745636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005745646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005745646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T08:41:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005745647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005745666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,678 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=45, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/63f88188936f40afac3f703bf3af78fd 2024-11-19T08:41:25,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/cd51077ac2c24fbd9b689b66c1a046a3 is 50, key is test_row_1/B:col10/1732005685179/Put/seqid=0 2024-11-19T08:41:25,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742145_1321 (size=9657) 2024-11-19T08:41:25,708 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/cd51077ac2c24fbd9b689b66c1a046a3 2024-11-19T08:41:25,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/b9e2c5c629a94313a977822ce86ea423 is 50, key is test_row_1/C:col10/1732005685179/Put/seqid=0 2024-11-19T08:41:25,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742146_1322 (size=9657) 2024-11-19T08:41:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005745946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005745956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005745966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005745967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:25,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005745986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,126 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/b9e2c5c629a94313a977822ce86ea423 2024-11-19T08:41:26,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/63f88188936f40afac3f703bf3af78fd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/63f88188936f40afac3f703bf3af78fd 2024-11-19T08:41:26,140 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/63f88188936f40afac3f703bf3af78fd, entries=100, sequenceid=45, filesize=21.8 K 2024-11-19T08:41:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/cd51077ac2c24fbd9b689b66c1a046a3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd51077ac2c24fbd9b689b66c1a046a3 2024-11-19T08:41:26,146 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd51077ac2c24fbd9b689b66c1a046a3, entries=100, sequenceid=45, filesize=9.4 K 2024-11-19T08:41:26,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/b9e2c5c629a94313a977822ce86ea423 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b9e2c5c629a94313a977822ce86ea423 2024-11-19T08:41:26,150 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b9e2c5c629a94313a977822ce86ea423, entries=100, sequenceid=45, filesize=9.4 K 2024-11-19T08:41:26,151 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 8f3ff379c750bd3082218a4d8d91a02f in 966ms, sequenceid=45, compaction requested=true 2024-11-19T08:41:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T08:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T08:41:26,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T08:41:26,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1210 sec 2024-11-19T08:41:26,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.1270 sec 2024-11-19T08:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T08:41:26,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:26,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T08:41:26,168 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:26,169 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:26,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T08:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T08:41:26,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T08:41:26,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:26,322 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-19T08:41:26,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e346484a8cac48f8bcafffada0abe64c_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005685312/Put/seqid=0 2024-11-19T08:41:26,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742147_1323 (size=12154) 2024-11-19T08:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:26,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005746475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005746475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005746476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005746476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T08:41:26,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005746505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005746596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005746596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:26,737 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e346484a8cac48f8bcafffada0abe64c_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e346484a8cac48f8bcafffada0abe64c_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:26,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d9263abd670c486599b977415811ea1f, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:26,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d9263abd670c486599b977415811ea1f is 175, key is test_row_0/A:col10/1732005685312/Put/seqid=0 2024-11-19T08:41:26,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742148_1324 (size=30955) 2024-11-19T08:41:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T08:41:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005746806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:26,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005746806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005747116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005747126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,155 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d9263abd670c486599b977415811ea1f 2024-11-19T08:41:27,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/99d2c4dd905e40e1b3a1aeaea8660f43 is 50, key is test_row_0/B:col10/1732005685312/Put/seqid=0 2024-11-19T08:41:27,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742149_1325 (size=12001) 2024-11-19T08:41:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T08:41:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005747496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005747496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,502 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,502 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005747516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,521 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,570 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/99d2c4dd905e40e1b3a1aeaea8660f43 2024-11-19T08:41:27,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/33436b69b77b440a90f05ed66d308772 is 50, key is test_row_0/C:col10/1732005685312/Put/seqid=0 2024-11-19T08:41:27,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742150_1326 (size=12001) 2024-11-19T08:41:27,580 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/33436b69b77b440a90f05ed66d308772 2024-11-19T08:41:27,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d9263abd670c486599b977415811ea1f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d9263abd670c486599b977415811ea1f 2024-11-19T08:41:27,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d9263abd670c486599b977415811ea1f, entries=150, sequenceid=76, filesize=30.2 K 2024-11-19T08:41:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/99d2c4dd905e40e1b3a1aeaea8660f43 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/99d2c4dd905e40e1b3a1aeaea8660f43 2024-11-19T08:41:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/99d2c4dd905e40e1b3a1aeaea8660f43, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T08:41:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/33436b69b77b440a90f05ed66d308772 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/33436b69b77b440a90f05ed66d308772 2024-11-19T08:41:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,598 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/33436b69b77b440a90f05ed66d308772, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T08:41:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for 8f3ff379c750bd3082218a4d8d91a02f in 1279ms, sequenceid=76, compaction requested=true 2024-11-19T08:41:27,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:27,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T08:41:27,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4330 sec 2024-11-19T08:41:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4370 sec 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:27,705 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:27,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:41:27,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111994cdc9ba2f534d3991bdf1424dd04515_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005687666/Put/seqid=0 2024-11-19T08:41:27,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742151_1327 (size=12154) 2024-11-19T08:41:27,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005747848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005747848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005747955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005747955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:28,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:28,139 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111994cdc9ba2f534d3991bdf1424dd04515_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994cdc9ba2f534d3991bdf1424dd04515_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:28,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/fed1991880224f059f08000320cbb2a9, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:28,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/fed1991880224f059f08000320cbb2a9 is 175, key is test_row_0/A:col10/1732005687666/Put/seqid=0 2024-11-19T08:41:28,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742152_1328 (size=30955) 2024-11-19T08:41:28,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/fed1991880224f059f08000320cbb2a9 2024-11-19T08:41:28,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/b889d027dfc3489c8a1d61d6ba6dfe29 is 50, key is test_row_0/B:col10/1732005687666/Put/seqid=0 2024-11-19T08:41:28,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742153_1329 (size=12001) 2024-11-19T08:41:28,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005748166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005748166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T08:41:28,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:28,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T08:41:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T08:41:28,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:28,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:28,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T08:41:28,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:28,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:28,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005748476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:28,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005748476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:28,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/b889d027dfc3489c8a1d61d6ba6dfe29 2024-11-19T08:41:28,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/6132dd68c9e8442882b7e1243b93afa2 is 50, key is test_row_0/C:col10/1732005687666/Put/seqid=0 2024-11-19T08:41:28,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742154_1330 (size=12001) 2024-11-19T08:41:28,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T08:41:28,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:28,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:28,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T08:41:28,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/6132dd68c9e8442882b7e1243b93afa2 2024-11-19T08:41:28,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/fed1991880224f059f08000320cbb2a9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/fed1991880224f059f08000320cbb2a9 2024-11-19T08:41:28,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/fed1991880224f059f08000320cbb2a9, entries=150, sequenceid=88, filesize=30.2 K 2024-11-19T08:41:28,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/b889d027dfc3489c8a1d61d6ba6dfe29 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/b889d027dfc3489c8a1d61d6ba6dfe29 2024-11-19T08:41:28,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/b889d027dfc3489c8a1d61d6ba6dfe29, entries=150, sequenceid=88, filesize=11.7 K 2024-11-19T08:41:28,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/6132dd68c9e8442882b7e1243b93afa2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/6132dd68c9e8442882b7e1243b93afa2 2024-11-19T08:41:28,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/6132dd68c9e8442882b7e1243b93afa2, entries=150, sequenceid=88, filesize=11.7 K 2024-11-19T08:41:28,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8f3ff379c750bd3082218a4d8d91a02f in 1274ms, sequenceid=88, compaction requested=true 2024-11-19T08:41:28,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:28,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:28,980 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T08:41:28,980 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T08:41:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:28,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:28,981 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T08:41:28,981 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/B is initiating minor compaction (all files) 2024-11-19T08:41:28,981 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/B in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,981 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/98c9994c8d754046bdf4ed8f1f718b9b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/749fadc57aa34916a06f9a149ac10375, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd51077ac2c24fbd9b689b66c1a046a3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/99d2c4dd905e40e1b3a1aeaea8660f43, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/b889d027dfc3489c8a1d61d6ba6dfe29] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=56.3 K 2024-11-19T08:41:28,981 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 146181 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T08:41:28,981 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/A is initiating minor compaction (all files) 2024-11-19T08:41:28,981 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/A in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,982 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/83875ee1b05d4313b6dcdc784698dc1b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/0273427c945849748edc154067de328f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/63f88188936f40afac3f703bf3af78fd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d9263abd670c486599b977415811ea1f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/fed1991880224f059f08000320cbb2a9] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=142.8 K 2024-11-19T08:41:28,982 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:28,982 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/83875ee1b05d4313b6dcdc784698dc1b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/0273427c945849748edc154067de328f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/63f88188936f40afac3f703bf3af78fd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d9263abd670c486599b977415811ea1f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/fed1991880224f059f08000320cbb2a9] 2024-11-19T08:41:28,982 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c9994c8d754046bdf4ed8f1f718b9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732005682866 2024-11-19T08:41:28,982 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83875ee1b05d4313b6dcdc784698dc1b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732005682866 2024-11-19T08:41:28,982 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0273427c945849748edc154067de328f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732005682956 2024-11-19T08:41:28,982 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 749fadc57aa34916a06f9a149ac10375, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732005682956 2024-11-19T08:41:28,983 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f88188936f40afac3f703bf3af78fd, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732005685179 2024-11-19T08:41:28,983 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting cd51077ac2c24fbd9b689b66c1a046a3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732005685179 2024-11-19T08:41:28,983 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9263abd670c486599b977415811ea1f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005685312 2024-11-19T08:41:28,983 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 99d2c4dd905e40e1b3a1aeaea8660f43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005685312 2024-11-19T08:41:28,983 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed1991880224f059f08000320cbb2a9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732005687666 2024-11-19T08:41:28,983 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b889d027dfc3489c8a1d61d6ba6dfe29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732005687666 2024-11-19T08:41:28,989 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:28,991 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:28,991 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/30f831dbedb243d2b00904fd09b26437 is 50, key is test_row_0/B:col10/1732005687666/Put/seqid=0 2024-11-19T08:41:28,992 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411192911123da31d4211b485c0a1a4ef36fe_8f3ff379c750bd3082218a4d8d91a02f store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:28,994 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411192911123da31d4211b485c0a1a4ef36fe_8f3ff379c750bd3082218a4d8d91a02f, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:28,995 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192911123da31d4211b485c0a1a4ef36fe_8f3ff379c750bd3082218a4d8d91a02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:28,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742155_1331 (size=12173) 2024-11-19T08:41:28,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742156_1332 (size=4469) 2024-11-19T08:41:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:29,004 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:29,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:41:29,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cda646cf60ec4957b17375960c6919a6_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005687846/Put/seqid=0 2024-11-19T08:41:29,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742157_1333 (size=14594) 2024-11-19T08:41:29,024 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:29,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cda646cf60ec4957b17375960c6919a6_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cda646cf60ec4957b17375960c6919a6_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:29,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c9f002af90c14a84b91d5f9f0e94cdbe, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:29,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c9f002af90c14a84b91d5f9f0e94cdbe is 175, key is test_row_0/A:col10/1732005687846/Put/seqid=0 2024-11-19T08:41:29,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742158_1334 (size=39549) 2024-11-19T08:41:29,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005749042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005749048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005749155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005749176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:29,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:29,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005749375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005749386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,401 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#A#compaction#283 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:29,401 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/ffcde04c390f451193f39907f4a972ff is 175, key is test_row_0/A:col10/1732005687666/Put/seqid=0 2024-11-19T08:41:29,402 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/30f831dbedb243d2b00904fd09b26437 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/30f831dbedb243d2b00904fd09b26437 2024-11-19T08:41:29,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742159_1335 (size=31127) 2024-11-19T08:41:29,407 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/B of 8f3ff379c750bd3082218a4d8d91a02f into 30f831dbedb243d2b00904fd09b26437(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:29,408 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:29,408 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/B, priority=11, startTime=1732005688980; duration=0sec 2024-11-19T08:41:29,408 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:29,408 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:B 2024-11-19T08:41:29,408 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T08:41:29,410 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T08:41:29,410 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/C is initiating minor compaction (all files) 2024-11-19T08:41:29,410 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/C in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,410 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b87d2ea737074a67bd3a47c298241734, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f2dff8a289b847f6af1ee51d89ea69f0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b9e2c5c629a94313a977822ce86ea423, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/33436b69b77b440a90f05ed66d308772, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/6132dd68c9e8442882b7e1243b93afa2] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=56.3 K 2024-11-19T08:41:29,410 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b87d2ea737074a67bd3a47c298241734, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732005682866 2024-11-19T08:41:29,411 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f2dff8a289b847f6af1ee51d89ea69f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732005682956 2024-11-19T08:41:29,411 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/ffcde04c390f451193f39907f4a972ff as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ffcde04c390f451193f39907f4a972ff 2024-11-19T08:41:29,411 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e2c5c629a94313a977822ce86ea423, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732005685179 2024-11-19T08:41:29,412 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 33436b69b77b440a90f05ed66d308772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005685312 2024-11-19T08:41:29,412 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6132dd68c9e8442882b7e1243b93afa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732005687666 2024-11-19T08:41:29,420 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/A of 8f3ff379c750bd3082218a4d8d91a02f into ffcde04c390f451193f39907f4a972ff(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:29,420 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:29,420 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/A, priority=11, startTime=1732005688979; duration=0sec 2024-11-19T08:41:29,420 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:29,420 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:A 2024-11-19T08:41:29,427 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:29,429 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/20234862f41a4430a2d3b936abb95bc1 is 50, key is test_row_0/C:col10/1732005687666/Put/seqid=0 2024-11-19T08:41:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742160_1336 (size=12173) 2024-11-19T08:41:29,436 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c9f002af90c14a84b91d5f9f0e94cdbe 2024-11-19T08:41:29,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/54a7e05f5b85424c8196623433f77e5e is 50, key is test_row_0/B:col10/1732005687846/Put/seqid=0 2024-11-19T08:41:29,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742161_1337 (size=12001) 2024-11-19T08:41:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T08:41:29,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005749516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,523 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005749526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,533 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005749545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,552 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005749696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005749706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:29,838 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/20234862f41a4430a2d3b936abb95bc1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/20234862f41a4430a2d3b936abb95bc1 2024-11-19T08:41:29,842 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/C of 8f3ff379c750bd3082218a4d8d91a02f into 20234862f41a4430a2d3b936abb95bc1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:29,842 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:29,842 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/C, priority=11, startTime=1732005688980; duration=0sec 2024-11-19T08:41:29,843 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:29,843 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:C 2024-11-19T08:41:29,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:29,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:29,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/54a7e05f5b85424c8196623433f77e5e 2024-11-19T08:41:29,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/045ed02a3f664e28b87b64d58978aa37 is 50, key is test_row_0/C:col10/1732005687846/Put/seqid=0 2024-11-19T08:41:29,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742162_1338 (size=12001) 2024-11-19T08:41:29,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/045ed02a3f664e28b87b64d58978aa37 2024-11-19T08:41:29,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c9f002af90c14a84b91d5f9f0e94cdbe as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c9f002af90c14a84b91d5f9f0e94cdbe 2024-11-19T08:41:29,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c9f002af90c14a84b91d5f9f0e94cdbe, entries=200, sequenceid=113, filesize=38.6 K 2024-11-19T08:41:29,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/54a7e05f5b85424c8196623433f77e5e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/54a7e05f5b85424c8196623433f77e5e 2024-11-19T08:41:29,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/54a7e05f5b85424c8196623433f77e5e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-19T08:41:29,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/045ed02a3f664e28b87b64d58978aa37 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/045ed02a3f664e28b87b64d58978aa37 2024-11-19T08:41:29,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/045ed02a3f664e28b87b64d58978aa37, entries=150, sequenceid=113, filesize=11.7 K 2024-11-19T08:41:29,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8f3ff379c750bd3082218a4d8d91a02f in 882ms, sequenceid=113, compaction requested=false 2024-11-19T08:41:29,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:30,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T08:41:30,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:30,000 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:41:30,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119afbce7bc068149f5a4a63433b628c983_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005689041/Put/seqid=0 2024-11-19T08:41:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742163_1339 (size=12154) 2024-11-19T08:41:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:30,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005750274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005750280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:30,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005750386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:30,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005750396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:30,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:30,415 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119afbce7bc068149f5a4a63433b628c983_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afbce7bc068149f5a4a63433b628c983_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:30,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c387818ab1ef488c815ec5dd97ffdb64, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c387818ab1ef488c815ec5dd97ffdb64 is 175, key is test_row_0/A:col10/1732005689041/Put/seqid=0 2024-11-19T08:41:30,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742164_1340 (size=30955) 2024-11-19T08:41:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T08:41:30,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005750596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005750606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:30,820 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c387818ab1ef488c815ec5dd97ffdb64 2024-11-19T08:41:30,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/8360533e3644469e8444e17dd85971a8 is 50, key is test_row_0/B:col10/1732005689041/Put/seqid=0 2024-11-19T08:41:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742165_1341 (size=12001) 2024-11-19T08:41:30,830 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/8360533e3644469e8444e17dd85971a8 2024-11-19T08:41:30,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/345f1fe0e51743ed8b2d3fcffd25fc3b is 50, key is test_row_0/C:col10/1732005689041/Put/seqid=0 2024-11-19T08:41:30,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742166_1342 (size=12001) 2024-11-19T08:41:30,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/345f1fe0e51743ed8b2d3fcffd25fc3b 2024-11-19T08:41:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/c387818ab1ef488c815ec5dd97ffdb64 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c387818ab1ef488c815ec5dd97ffdb64 2024-11-19T08:41:30,854 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c387818ab1ef488c815ec5dd97ffdb64, entries=150, sequenceid=127, filesize=30.2 K 2024-11-19T08:41:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/8360533e3644469e8444e17dd85971a8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8360533e3644469e8444e17dd85971a8 2024-11-19T08:41:30,857 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8360533e3644469e8444e17dd85971a8, entries=150, sequenceid=127, filesize=11.7 K 2024-11-19T08:41:30,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/345f1fe0e51743ed8b2d3fcffd25fc3b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/345f1fe0e51743ed8b2d3fcffd25fc3b 2024-11-19T08:41:30,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/345f1fe0e51743ed8b2d3fcffd25fc3b, entries=150, sequenceid=127, filesize=11.7 K 2024-11-19T08:41:30,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8f3ff379c750bd3082218a4d8d91a02f in 862ms, sequenceid=127, compaction requested=true 2024-11-19T08:41:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T08:41:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T08:41:30,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T08:41:30,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5450 sec 2024-11-19T08:41:30,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.5510 sec 2024-11-19T08:41:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:30,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:30,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:41:30,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111974789ac435584ed1b23624cab665ce2f_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005690923/Put/seqid=0 2024-11-19T08:41:30,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742167_1343 (size=14794) 2024-11-19T08:41:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005750949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:30,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005750950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:31,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005751055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:31,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005751066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:31,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005751266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:31,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005751276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:31,333 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:31,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111974789ac435584ed1b23624cab665ce2f_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111974789ac435584ed1b23624cab665ce2f_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:31,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/b2a048d9811b40c88588d908ee52037a, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:31,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/b2a048d9811b40c88588d908ee52037a is 175, key is test_row_0/A:col10/1732005690923/Put/seqid=0 2024-11-19T08:41:31,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742168_1344 (size=39749) 2024-11-19T08:41:31,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005751586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:31,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005751596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:31,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:31,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/b2a048d9811b40c88588d908ee52037a 2024-11-19T08:41:31,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/3882d1290fa54e5caefff7e3a29f852a is 50, key is test_row_0/B:col10/1732005690923/Put/seqid=0 2024-11-19T08:41:31,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742169_1345 (size=12151) 2024-11-19T08:41:31,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/3882d1290fa54e5caefff7e3a29f852a 2024-11-19T08:41:31,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/54ac0126982d484f818838c884339d46 is 50, key is test_row_0/C:col10/1732005690923/Put/seqid=0 2024-11-19T08:41:31,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742170_1346 (size=12151) 2024-11-19T08:41:32,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005752106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:32,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005752115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:32,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/54ac0126982d484f818838c884339d46 2024-11-19T08:41:32,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/b2a048d9811b40c88588d908ee52037a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/b2a048d9811b40c88588d908ee52037a 2024-11-19T08:41:32,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/b2a048d9811b40c88588d908ee52037a, entries=200, sequenceid=153, filesize=38.8 K 2024-11-19T08:41:32,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/3882d1290fa54e5caefff7e3a29f852a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/3882d1290fa54e5caefff7e3a29f852a 2024-11-19T08:41:32,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/3882d1290fa54e5caefff7e3a29f852a, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T08:41:32,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/54ac0126982d484f818838c884339d46 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/54ac0126982d484f818838c884339d46 2024-11-19T08:41:32,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/54ac0126982d484f818838c884339d46, entries=150, sequenceid=153, filesize=11.9 K 2024-11-19T08:41:32,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8f3ff379c750bd3082218a4d8d91a02f in 1312ms, sequenceid=153, compaction requested=true 2024-11-19T08:41:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:32,236 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:41:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:32,236 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:41:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:32,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:32,237 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141380 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:41:32,237 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:41:32,237 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/A is initiating minor compaction (all files) 2024-11-19T08:41:32,237 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/B is initiating minor compaction (all files) 2024-11-19T08:41:32,237 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/B in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:32,237 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/A in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:32,237 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/30f831dbedb243d2b00904fd09b26437, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/54a7e05f5b85424c8196623433f77e5e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8360533e3644469e8444e17dd85971a8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/3882d1290fa54e5caefff7e3a29f852a] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=47.2 K 2024-11-19T08:41:32,237 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ffcde04c390f451193f39907f4a972ff, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c9f002af90c14a84b91d5f9f0e94cdbe, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c387818ab1ef488c815ec5dd97ffdb64, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/b2a048d9811b40c88588d908ee52037a] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=138.1 K 2024-11-19T08:41:32,237 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:32,237 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ffcde04c390f451193f39907f4a972ff, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c9f002af90c14a84b91d5f9f0e94cdbe, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c387818ab1ef488c815ec5dd97ffdb64, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/b2a048d9811b40c88588d908ee52037a] 2024-11-19T08:41:32,238 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 30f831dbedb243d2b00904fd09b26437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732005687666 2024-11-19T08:41:32,238 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffcde04c390f451193f39907f4a972ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732005687666 2024-11-19T08:41:32,238 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 54a7e05f5b85424c8196623433f77e5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732005687830 2024-11-19T08:41:32,238 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8360533e3644469e8444e17dd85971a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732005689027 2024-11-19T08:41:32,238 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f002af90c14a84b91d5f9f0e94cdbe, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732005687798 2024-11-19T08:41:32,238 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 3882d1290fa54e5caefff7e3a29f852a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732005690256 2024-11-19T08:41:32,238 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c387818ab1ef488c815ec5dd97ffdb64, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732005689027 2024-11-19T08:41:32,239 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2a048d9811b40c88588d908ee52037a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732005690256 2024-11-19T08:41:32,246 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:32,246 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/a22f153dc66b4df58c336cd834a6308a is 50, key is test_row_0/B:col10/1732005690923/Put/seqid=0 2024-11-19T08:41:32,247 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:32,259 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411192374fad52903403fa3027d14c35b2280_8f3ff379c750bd3082218a4d8d91a02f store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:32,262 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411192374fad52903403fa3027d14c35b2280_8f3ff379c750bd3082218a4d8d91a02f, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:32,262 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192374fad52903403fa3027d14c35b2280_8f3ff379c750bd3082218a4d8d91a02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:32,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742171_1347 (size=12459) 2024-11-19T08:41:32,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742172_1348 (size=4469) 2024-11-19T08:41:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T08:41:32,476 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:32,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T08:41:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T08:41:32,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:32,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:32,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T08:41:32,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T08:41:32,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:32,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T08:41:32,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f27239fb3612405484cba86d7407d97c_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005690948/Put/seqid=0 2024-11-19T08:41:32,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742173_1349 (size=12304) 2024-11-19T08:41:32,666 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/a22f153dc66b4df58c336cd834a6308a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/a22f153dc66b4df58c336cd834a6308a 2024-11-19T08:41:32,668 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:32,668 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/f5fb24cea9cc43eda6b20918470f7ae1 is 175, key is test_row_0/A:col10/1732005690923/Put/seqid=0 2024-11-19T08:41:32,671 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/B of 8f3ff379c750bd3082218a4d8d91a02f into a22f153dc66b4df58c336cd834a6308a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:32,671 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:32,671 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/B, priority=12, startTime=1732005692236; duration=0sec 2024-11-19T08:41:32,672 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:32,672 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:B 2024-11-19T08:41:32,672 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:41:32,673 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:41:32,673 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/C is initiating minor compaction (all files) 2024-11-19T08:41:32,673 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/C in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:32,673 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/20234862f41a4430a2d3b936abb95bc1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/045ed02a3f664e28b87b64d58978aa37, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/345f1fe0e51743ed8b2d3fcffd25fc3b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/54ac0126982d484f818838c884339d46] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=47.2 K 2024-11-19T08:41:32,673 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 20234862f41a4430a2d3b936abb95bc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732005687666 2024-11-19T08:41:32,673 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 045ed02a3f664e28b87b64d58978aa37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732005687830 2024-11-19T08:41:32,673 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 345f1fe0e51743ed8b2d3fcffd25fc3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732005689027 2024-11-19T08:41:32,674 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ac0126982d484f818838c884339d46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732005690256 2024-11-19T08:41:32,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742174_1350 (size=31413) 2024-11-19T08:41:32,685 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/f5fb24cea9cc43eda6b20918470f7ae1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/f5fb24cea9cc43eda6b20918470f7ae1 2024-11-19T08:41:32,689 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:32,689 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/91d78a4a7475478d859f472c8c5b1c5f is 50, key is test_row_0/C:col10/1732005690923/Put/seqid=0 2024-11-19T08:41:32,690 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/A of 8f3ff379c750bd3082218a4d8d91a02f into f5fb24cea9cc43eda6b20918470f7ae1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:32,690 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:32,690 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/A, priority=12, startTime=1732005692236; duration=0sec 2024-11-19T08:41:32,690 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:32,690 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:A 2024-11-19T08:41:32,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742175_1351 (size=12459) 2024-11-19T08:41:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T08:41:33,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:33,043 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f27239fb3612405484cba86d7407d97c_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f27239fb3612405484cba86d7407d97c_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:33,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a557d7bf4aad4af48d2b76f5e2cb492c, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:33,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a557d7bf4aad4af48d2b76f5e2cb492c is 175, key is test_row_0/A:col10/1732005690948/Put/seqid=0 2024-11-19T08:41:33,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742176_1352 (size=31105) 2024-11-19T08:41:33,100 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/91d78a4a7475478d859f472c8c5b1c5f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/91d78a4a7475478d859f472c8c5b1c5f 2024-11-19T08:41:33,104 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/C of 8f3ff379c750bd3082218a4d8d91a02f into 91d78a4a7475478d859f472c8c5b1c5f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:33,104 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:33,104 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/C, priority=12, startTime=1732005692236; duration=0sec 2024-11-19T08:41:33,104 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:33,104 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:C 2024-11-19T08:41:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T08:41:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:33,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005753210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005753210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005753326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005753326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,448 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a557d7bf4aad4af48d2b76f5e2cb492c 2024-11-19T08:41:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/f29df51172b64b22867e2bed5a8d974e is 50, key is test_row_0/B:col10/1732005690948/Put/seqid=0 2024-11-19T08:41:33,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742177_1353 (size=12151) 2024-11-19T08:41:33,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005753536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005753536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43282 deadline: 1732005753546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,553 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43242 deadline: 1732005753555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,562 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43196 deadline: 1732005753586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,593 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in 8f3ff379c750bd3082218a4d8d91a02f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T08:41:33,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005753846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,858 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/f29df51172b64b22867e2bed5a8d974e 2024-11-19T08:41:33,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005753856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/67c33f7a6411487fa181cb15db06d306 is 50, key is test_row_0/C:col10/1732005690948/Put/seqid=0 2024-11-19T08:41:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742178_1354 (size=12151) 2024-11-19T08:41:33,869 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/67c33f7a6411487fa181cb15db06d306 2024-11-19T08:41:33,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a557d7bf4aad4af48d2b76f5e2cb492c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a557d7bf4aad4af48d2b76f5e2cb492c 2024-11-19T08:41:33,875 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a557d7bf4aad4af48d2b76f5e2cb492c, entries=150, sequenceid=163, filesize=30.4 K 2024-11-19T08:41:33,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T08:41:33,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/f29df51172b64b22867e2bed5a8d974e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/f29df51172b64b22867e2bed5a8d974e 2024-11-19T08:41:33,879 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/f29df51172b64b22867e2bed5a8d974e, entries=150, sequenceid=163, filesize=11.9 K 2024-11-19T08:41:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/67c33f7a6411487fa181cb15db06d306 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/67c33f7a6411487fa181cb15db06d306 2024-11-19T08:41:33,883 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/67c33f7a6411487fa181cb15db06d306, entries=150, sequenceid=163, filesize=11.9 K 2024-11-19T08:41:33,884 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 8f3ff379c750bd3082218a4d8d91a02f in 1253ms, sequenceid=163, compaction requested=false 2024-11-19T08:41:33,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:33,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:33,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T08:41:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T08:41:33,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T08:41:33,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-19T08:41:33,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4120 sec 2024-11-19T08:41:34,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:34,370 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:34,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-19T08:41:34,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194132f830b5ce40549a42eaf9f0739822_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005693194/Put/seqid=0 2024-11-19T08:41:34,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742179_1355 (size=17284) 2024-11-19T08:41:34,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:34,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194132f830b5ce40549a42eaf9f0739822_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194132f830b5ce40549a42eaf9f0739822_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:34,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a08dceb0d5e0482c89e442ff899cc67d, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:34,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a08dceb0d5e0482c89e442ff899cc67d is 175, key is test_row_0/A:col10/1732005693194/Put/seqid=0 2024-11-19T08:41:34,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005754384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742180_1356 (size=48389) 2024-11-19T08:41:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005754385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:34,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a08dceb0d5e0482c89e442ff899cc67d 2024-11-19T08:41:34,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/126f60d50af446efa760dbe11f18f320 is 50, key is test_row_0/B:col10/1732005693194/Put/seqid=0 2024-11-19T08:41:34,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742181_1357 (size=12151) 2024-11-19T08:41:34,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005754496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:34,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005754506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T08:41:34,626 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:34,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-19T08:41:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T08:41:34,628 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:34,629 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:34,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:34,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005754706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:34,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005754716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:34,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T08:41:34,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T08:41:34,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:34,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:34,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:34,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:34,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:34,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/126f60d50af446efa760dbe11f18f320 2024-11-19T08:41:34,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/0ca3ad977e9b434ca3d3df02730e68ca is 50, key is test_row_0/C:col10/1732005693194/Put/seqid=0 2024-11-19T08:41:34,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742182_1358 (size=12151) 2024-11-19T08:41:34,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T08:41:34,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:34,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:34,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:34,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:34,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T08:41:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005755016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:35,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005755025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:35,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T08:41:35,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:35,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:35,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:35,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:35,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:35,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/0ca3ad977e9b434ca3d3df02730e68ca 2024-11-19T08:41:35,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/a08dceb0d5e0482c89e442ff899cc67d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a08dceb0d5e0482c89e442ff899cc67d 2024-11-19T08:41:35,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a08dceb0d5e0482c89e442ff899cc67d, entries=250, sequenceid=195, filesize=47.3 K 2024-11-19T08:41:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/126f60d50af446efa760dbe11f18f320 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/126f60d50af446efa760dbe11f18f320 2024-11-19T08:41:35,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/126f60d50af446efa760dbe11f18f320, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T08:41:35,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/0ca3ad977e9b434ca3d3df02730e68ca as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/0ca3ad977e9b434ca3d3df02730e68ca 2024-11-19T08:41:35,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/0ca3ad977e9b434ca3d3df02730e68ca, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T08:41:35,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 8f3ff379c750bd3082218a4d8d91a02f in 864ms, sequenceid=195, compaction requested=true 2024-11-19T08:41:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:35,234 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:35,234 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:35,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:35,235 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:35,235 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:35,235 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/B is initiating minor compaction (all files) 2024-11-19T08:41:35,235 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/A is initiating minor compaction (all files) 2024-11-19T08:41:35,235 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/A in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:35,235 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/B in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:35,235 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/a22f153dc66b4df58c336cd834a6308a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/f29df51172b64b22867e2bed5a8d974e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/126f60d50af446efa760dbe11f18f320] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=35.9 K 2024-11-19T08:41:35,235 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/f5fb24cea9cc43eda6b20918470f7ae1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a557d7bf4aad4af48d2b76f5e2cb492c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a08dceb0d5e0482c89e442ff899cc67d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=108.3 K 2024-11-19T08:41:35,235 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:35,235 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/f5fb24cea9cc43eda6b20918470f7ae1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a557d7bf4aad4af48d2b76f5e2cb492c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a08dceb0d5e0482c89e442ff899cc67d] 2024-11-19T08:41:35,236 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a22f153dc66b4df58c336cd834a6308a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732005690256 2024-11-19T08:41:35,236 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5fb24cea9cc43eda6b20918470f7ae1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732005690256 2024-11-19T08:41:35,236 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f29df51172b64b22867e2bed5a8d974e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732005690937 2024-11-19T08:41:35,236 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a557d7bf4aad4af48d2b76f5e2cb492c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732005690937 2024-11-19T08:41:35,236 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08dceb0d5e0482c89e442ff899cc67d, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732005693177 2024-11-19T08:41:35,236 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 126f60d50af446efa760dbe11f18f320, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732005693194 2024-11-19T08:41:35,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T08:41:35,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:35,240 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T08:41:35,245 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:35,246 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/8a7d5df5759c48978e025ec439fb0467 is 50, key is test_row_0/B:col10/1732005693194/Put/seqid=0 2024-11-19T08:41:35,255 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T08:41:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111975e4a252c3454ce38a23db6530e4f82a_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005694382/Put/seqid=0 2024-11-19T08:41:35,267 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111911ab570faa1a48b8b81f9c80d1f84886_8f3ff379c750bd3082218a4d8d91a02f store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:35,268 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111911ab570faa1a48b8b81f9c80d1f84886_8f3ff379c750bd3082218a4d8d91a02f, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:35,268 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111911ab570faa1a48b8b81f9c80d1f84886_8f3ff379c750bd3082218a4d8d91a02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:35,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742183_1359 (size=12561) 2024-11-19T08:41:35,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742184_1360 (size=12304) 2024-11-19T08:41:35,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742185_1361 (size=4469) 2024-11-19T08:41:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:35,288 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#A#compaction#304 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:35,288 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d2b0bc7c1b804a75b50a930686c10717 is 175, key is test_row_0/A:col10/1732005693194/Put/seqid=0 2024-11-19T08:41:35,293 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111975e4a252c3454ce38a23db6530e4f82a_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111975e4a252c3454ce38a23db6530e4f82a_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/e7a4ecb1bcc045cc9e18215606ae4347, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/e7a4ecb1bcc045cc9e18215606ae4347 is 175, key is test_row_0/A:col10/1732005694382/Put/seqid=0 2024-11-19T08:41:35,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742186_1362 (size=31515) 2024-11-19T08:41:35,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742187_1363 (size=31105) 2024-11-19T08:41:35,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:35,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005755627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:35,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005755633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:35,680 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/8a7d5df5759c48978e025ec439fb0467 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8a7d5df5759c48978e025ec439fb0467 2024-11-19T08:41:35,684 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/B of 8f3ff379c750bd3082218a4d8d91a02f into 8a7d5df5759c48978e025ec439fb0467(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:35,684 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:35,684 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/B, priority=13, startTime=1732005695234; duration=0sec 2024-11-19T08:41:35,684 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:35,684 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:B 2024-11-19T08:41:35,684 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:35,685 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:35,685 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/C is initiating minor compaction (all files) 2024-11-19T08:41:35,685 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/C in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:35,685 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/91d78a4a7475478d859f472c8c5b1c5f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/67c33f7a6411487fa181cb15db06d306, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/0ca3ad977e9b434ca3d3df02730e68ca] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=35.9 K 2024-11-19T08:41:35,685 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d78a4a7475478d859f472c8c5b1c5f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732005690256 2024-11-19T08:41:35,685 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c33f7a6411487fa181cb15db06d306, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732005690937 2024-11-19T08:41:35,686 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca3ad977e9b434ca3d3df02730e68ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732005693194 2024-11-19T08:41:35,691 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#C#compaction#306 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:35,691 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/3b93d03fe0944eccbb1ee375b801a9e9 is 50, key is test_row_0/C:col10/1732005693194/Put/seqid=0 2024-11-19T08:41:35,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742188_1364 (size=12561) 2024-11-19T08:41:35,707 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d2b0bc7c1b804a75b50a930686c10717 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d2b0bc7c1b804a75b50a930686c10717 2024-11-19T08:41:35,710 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/A of 8f3ff379c750bd3082218a4d8d91a02f into d2b0bc7c1b804a75b50a930686c10717(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:35,710 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:35,711 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/A, priority=13, startTime=1732005695234; duration=0sec 2024-11-19T08:41:35,711 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:35,711 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:A 2024-11-19T08:41:35,711 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/e7a4ecb1bcc045cc9e18215606ae4347 2024-11-19T08:41:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/26ccbd28e3624cd680af037eeffd678c is 50, key is test_row_0/B:col10/1732005694382/Put/seqid=0 2024-11-19T08:41:35,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742189_1365 (size=12151) 2024-11-19T08:41:35,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005755745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005755746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T08:41:35,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005755955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:35,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005755966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:36,099 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/3b93d03fe0944eccbb1ee375b801a9e9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/3b93d03fe0944eccbb1ee375b801a9e9 2024-11-19T08:41:36,103 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/C of 8f3ff379c750bd3082218a4d8d91a02f into 3b93d03fe0944eccbb1ee375b801a9e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:36,103 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:36,103 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/C, priority=13, startTime=1732005695234; duration=0sec 2024-11-19T08:41:36,103 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:36,103 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:C 2024-11-19T08:41:36,120 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/26ccbd28e3624cd680af037eeffd678c 2024-11-19T08:41:36,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/89f0fd68d6c9416fbd6049a11bd9467b is 50, key is test_row_0/C:col10/1732005694382/Put/seqid=0 2024-11-19T08:41:36,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742190_1366 (size=12151) 2024-11-19T08:41:36,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005756266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:36,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005756276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:36,534 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/89f0fd68d6c9416fbd6049a11bd9467b 2024-11-19T08:41:36,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/e7a4ecb1bcc045cc9e18215606ae4347 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/e7a4ecb1bcc045cc9e18215606ae4347 2024-11-19T08:41:36,541 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/e7a4ecb1bcc045cc9e18215606ae4347, entries=150, sequenceid=202, filesize=30.4 K 2024-11-19T08:41:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/26ccbd28e3624cd680af037eeffd678c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/26ccbd28e3624cd680af037eeffd678c 2024-11-19T08:41:36,544 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/26ccbd28e3624cd680af037eeffd678c, entries=150, sequenceid=202, filesize=11.9 K 2024-11-19T08:41:36,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/89f0fd68d6c9416fbd6049a11bd9467b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/89f0fd68d6c9416fbd6049a11bd9467b 2024-11-19T08:41:36,549 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/89f0fd68d6c9416fbd6049a11bd9467b, entries=150, sequenceid=202, filesize=11.9 K 2024-11-19T08:41:36,549 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 8f3ff379c750bd3082218a4d8d91a02f in 1310ms, sequenceid=202, compaction requested=false 2024-11-19T08:41:36,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:36,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-19T08:41:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-19T08:41:36,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T08:41:36,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9230 sec 2024-11-19T08:41:36,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9270 sec 2024-11-19T08:41:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T08:41:36,776 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:36,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-19T08:41:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T08:41:36,778 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:36,778 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:36,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:36,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:36,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-19T08:41:36,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111949a3016ee3c742ada720117d9ccea150_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005696790/Put/seqid=0 2024-11-19T08:41:36,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005756795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:36,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:36,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742191_1367 (size=17284) 2024-11-19T08:41:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005756800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:36,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T08:41:36,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005756906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:36,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005756915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:36,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:36,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T08:41:37,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005757116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:37,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005757125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:37,202 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:37,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111949a3016ee3c742ada720117d9ccea150_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111949a3016ee3c742ada720117d9ccea150_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:37,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/37c888a81da34d0aa88c7afd0388910b, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:37,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/37c888a81da34d0aa88c7afd0388910b is 175, key is test_row_0/A:col10/1732005696790/Put/seqid=0 2024-11-19T08:41:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742192_1368 (size=48389) 2024-11-19T08:41:37,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:37,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:37,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T08:41:37,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005757426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:37,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005757436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:37,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/37c888a81da34d0aa88c7afd0388910b 2024-11-19T08:41:37,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/be8adbe8042e45a39655a9811de90103 is 50, key is test_row_0/B:col10/1732005696790/Put/seqid=0 2024-11-19T08:41:37,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742193_1369 (size=12151) 2024-11-19T08:41:37,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:37,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T08:41:37,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005757936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:37,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005757956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:37,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:37,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/be8adbe8042e45a39655a9811de90103 2024-11-19T08:41:38,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/74b3cf3390e64b66af85f4b43f5b1d4f is 50, key is test_row_0/C:col10/1732005696790/Put/seqid=0 2024-11-19T08:41:38,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742194_1370 (size=12151) 2024-11-19T08:41:38,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:38,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/74b3cf3390e64b66af85f4b43f5b1d4f 2024-11-19T08:41:38,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/37c888a81da34d0aa88c7afd0388910b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/37c888a81da34d0aa88c7afd0388910b 2024-11-19T08:41:38,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/37c888a81da34d0aa88c7afd0388910b, entries=250, sequenceid=236, filesize=47.3 K 2024-11-19T08:41:38,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/be8adbe8042e45a39655a9811de90103 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/be8adbe8042e45a39655a9811de90103 2024-11-19T08:41:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/be8adbe8042e45a39655a9811de90103, entries=150, sequenceid=236, filesize=11.9 K 2024-11-19T08:41:38,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/74b3cf3390e64b66af85f4b43f5b1d4f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/74b3cf3390e64b66af85f4b43f5b1d4f 2024-11-19T08:41:38,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/74b3cf3390e64b66af85f4b43f5b1d4f, entries=150, sequenceid=236, filesize=11.9 K 2024-11-19T08:41:38,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 8f3ff379c750bd3082218a4d8d91a02f in 1657ms, sequenceid=236, compaction requested=true 2024-11-19T08:41:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:38,447 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:38,447 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:38,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,448 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:38,448 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/B is initiating minor compaction (all files) 2024-11-19T08:41:38,448 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/B in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,449 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8a7d5df5759c48978e025ec439fb0467, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/26ccbd28e3624cd680af037eeffd678c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/be8adbe8042e45a39655a9811de90103] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=36.0 K 2024-11-19T08:41:38,449 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:38,449 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/A is initiating minor compaction (all files) 2024-11-19T08:41:38,449 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/A in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,449 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d2b0bc7c1b804a75b50a930686c10717, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/e7a4ecb1bcc045cc9e18215606ae4347, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/37c888a81da34d0aa88c7afd0388910b] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=108.4 K 2024-11-19T08:41:38,449 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,449 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d2b0bc7c1b804a75b50a930686c10717, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/e7a4ecb1bcc045cc9e18215606ae4347, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/37c888a81da34d0aa88c7afd0388910b] 2024-11-19T08:41:38,449 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7d5df5759c48978e025ec439fb0467, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732005693194 2024-11-19T08:41:38,449 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b0bc7c1b804a75b50a930686c10717, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732005693194 2024-11-19T08:41:38,449 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ccbd28e3624cd680af037eeffd678c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732005694378 2024-11-19T08:41:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,450 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a4ecb1bcc045cc9e18215606ae4347, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732005694378 2024-11-19T08:41:38,450 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting be8adbe8042e45a39655a9811de90103, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732005695603 2024-11-19T08:41:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,450 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37c888a81da34d0aa88c7afd0388910b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732005695603 2024-11-19T08:41:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T08:41:38,458 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,459 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-19T08:41:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,468 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#B#compaction#313 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,468 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/4cc20e971ce84837813de313f0ac283e is 50, key is test_row_0/B:col10/1732005696790/Put/seqid=0 2024-11-19T08:41:38,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,473 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119698ac0edee234e959ad607bcb12663e4_8f3ff379c750bd3082218a4d8d91a02f store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,477 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119698ac0edee234e959ad607bcb12663e4_8f3ff379c750bd3082218a4d8d91a02f, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:38,477 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119698ac0edee234e959ad607bcb12663e4_8f3ff379c750bd3082218a4d8d91a02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742195_1371 (size=4469) 2024-11-19T08:41:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742196_1372 (size=12663) 2024-11-19T08:41:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,490 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#A#compaction#312 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,490 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/96ee329ddae74681845392502d171a72 is 175, key is test_row_0/A:col10/1732005696790/Put/seqid=0 2024-11-19T08:41:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111928c43a06da2c48069f9025a54ed2992a_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005696798/Put/seqid=0 2024-11-19T08:41:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742198_1374 (size=9814) 2024-11-19T08:41:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742197_1373 (size=31617) 2024-11-19T08:41:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,537 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/96ee329ddae74681845392502d171a72 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/96ee329ddae74681845392502d171a72 2024-11-19T08:41:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,543 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/A of 8f3ff379c750bd3082218a4d8d91a02f into 96ee329ddae74681845392502d171a72(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:38,543 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:38,543 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/A, priority=13, startTime=1732005698447; duration=0sec 2024-11-19T08:41:38,543 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:38,543 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:A 2024-11-19T08:41:38,543 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,544 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:38,544 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/C is initiating minor compaction (all files) 2024-11-19T08:41:38,544 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/C in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,544 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/3b93d03fe0944eccbb1ee375b801a9e9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/89f0fd68d6c9416fbd6049a11bd9467b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/74b3cf3390e64b66af85f4b43f5b1d4f] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=36.0 K 2024-11-19T08:41:38,545 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b93d03fe0944eccbb1ee375b801a9e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732005693194 2024-11-19T08:41:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,545 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f0fd68d6c9416fbd6049a11bd9467b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732005694378 2024-11-19T08:41:38,545 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b3cf3390e64b66af85f4b43f5b1d4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732005695603 2024-11-19T08:41:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,555 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,556 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/7033c0d8fb994a06977d2652a6acaa00 is 50, key is test_row_0/C:col10/1732005696790/Put/seqid=0 2024-11-19T08:41:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742199_1375 (size=12663) 2024-11-19T08:41:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,891 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/4cc20e971ce84837813de313f0ac283e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4cc20e971ce84837813de313f0ac283e 2024-11-19T08:41:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,895 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/B of 8f3ff379c750bd3082218a4d8d91a02f into 4cc20e971ce84837813de313f0ac283e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:38,895 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:38,895 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/B, priority=13, startTime=1732005698447; duration=0sec 2024-11-19T08:41:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,896 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:38,896 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:B 2024-11-19T08:41:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111928c43a06da2c48069f9025a54ed2992a_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111928c43a06da2c48069f9025a54ed2992a_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/2feca0e9564f4f70a7ab61abe9d4940f, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/2feca0e9564f4f70a7ab61abe9d4940f is 175, key is test_row_0/A:col10/1732005696798/Put/seqid=0 2024-11-19T08:41:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T08:41:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742200_1376 (size=22461) 2024-11-19T08:41:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,927 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/2feca0e9564f4f70a7ab61abe9d4940f 2024-11-19T08:41:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/2ca63d65a73447bfb88f2b7a864364e8 is 50, key is test_row_0/B:col10/1732005696798/Put/seqid=0 2024-11-19T08:41:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742201_1377 (size=9757) 2024-11-19T08:41:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,945 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/2ca63d65a73447bfb88f2b7a864364e8 2024-11-19T08:41:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/620baace61e84b588ff9cf48c9973c95 is 50, key is test_row_0/C:col10/1732005696798/Put/seqid=0 2024-11-19T08:41:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742202_1378 (size=9757) 2024-11-19T08:41:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,961 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/620baace61e84b588ff9cf48c9973c95 2024-11-19T08:41:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/2feca0e9564f4f70a7ab61abe9d4940f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/2feca0e9564f4f70a7ab61abe9d4940f 2024-11-19T08:41:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,968 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/7033c0d8fb994a06977d2652a6acaa00 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/7033c0d8fb994a06977d2652a6acaa00 2024-11-19T08:41:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,970 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/2feca0e9564f4f70a7ab61abe9d4940f, entries=100, sequenceid=241, filesize=21.9 K 2024-11-19T08:41:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/2ca63d65a73447bfb88f2b7a864364e8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/2ca63d65a73447bfb88f2b7a864364e8 2024-11-19T08:41:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,975 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/C of 8f3ff379c750bd3082218a4d8d91a02f into 7033c0d8fb994a06977d2652a6acaa00(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:38,975 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:38,976 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/C, priority=13, startTime=1732005698447; duration=0sec 2024-11-19T08:41:38,976 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/2ca63d65a73447bfb88f2b7a864364e8, entries=100, sequenceid=241, filesize=9.5 K 2024-11-19T08:41:38,976 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:38,976 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:C 2024-11-19T08:41:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/620baace61e84b588ff9cf48c9973c95 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/620baace61e84b588ff9cf48c9973c95 2024-11-19T08:41:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,982 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/620baace61e84b588ff9cf48c9973c95, entries=100, sequenceid=241, filesize=9.5 K 2024-11-19T08:41:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,982 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 8f3ff379c750bd3082218a4d8d91a02f in 523ms, sequenceid=241, compaction requested=false 2024-11-19T08:41:38,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:38,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-19T08:41:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-19T08:41:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T08:41:38,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-11-19T08:41:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.2100 sec 2024-11-19T08:41:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:39,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197bbe885858eb4ef2b9f25f6ab09f1cf7_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005698983/Put/seqid=0 2024-11-19T08:41:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742203_1379 (size=14794) 2024-11-19T08:41:39,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005759243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005759245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005759366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005759366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:39,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197bbe885858eb4ef2b9f25f6ab09f1cf7_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197bbe885858eb4ef2b9f25f6ab09f1cf7_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:39,545 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/29aca2cb06bc43a986490295d34690fd, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:39,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/29aca2cb06bc43a986490295d34690fd is 175, key is test_row_0/A:col10/1732005698983/Put/seqid=0 2024-11-19T08:41:39,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742204_1380 (size=39749) 2024-11-19T08:41:39,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005759576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005759586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005759886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005759896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:39,951 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/29aca2cb06bc43a986490295d34690fd 2024-11-19T08:41:39,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/00aa6d4828644246bd4f4c530567bc60 is 50, key is test_row_0/B:col10/1732005698983/Put/seqid=0 2024-11-19T08:41:39,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742205_1381 (size=12151) 2024-11-19T08:41:40,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/00aa6d4828644246bd4f4c530567bc60 2024-11-19T08:41:40,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/c8128ae53dd646d98d402e43e6010ed7 is 50, key is test_row_0/C:col10/1732005698983/Put/seqid=0 2024-11-19T08:41:40,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742206_1382 (size=12151) 2024-11-19T08:41:40,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005760406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:40,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005760416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:40,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/c8128ae53dd646d98d402e43e6010ed7 2024-11-19T08:41:40,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/29aca2cb06bc43a986490295d34690fd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/29aca2cb06bc43a986490295d34690fd 2024-11-19T08:41:40,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/29aca2cb06bc43a986490295d34690fd, entries=200, sequenceid=255, filesize=38.8 K 2024-11-19T08:41:40,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/00aa6d4828644246bd4f4c530567bc60 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/00aa6d4828644246bd4f4c530567bc60 2024-11-19T08:41:40,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/00aa6d4828644246bd4f4c530567bc60, entries=150, sequenceid=255, filesize=11.9 K 2024-11-19T08:41:40,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/c8128ae53dd646d98d402e43e6010ed7 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/c8128ae53dd646d98d402e43e6010ed7 2024-11-19T08:41:40,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/c8128ae53dd646d98d402e43e6010ed7, entries=150, sequenceid=255, filesize=11.9 K 2024-11-19T08:41:40,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8f3ff379c750bd3082218a4d8d91a02f in 1675ms, sequenceid=255, compaction requested=true 2024-11-19T08:41:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:40,793 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:40,793 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:40,794 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:40,794 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:40,794 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/B is initiating minor compaction (all files) 2024-11-19T08:41:40,794 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/A is initiating minor compaction (all files) 2024-11-19T08:41:40,794 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/B in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:40,794 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/A in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:40,794 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4cc20e971ce84837813de313f0ac283e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/2ca63d65a73447bfb88f2b7a864364e8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/00aa6d4828644246bd4f4c530567bc60] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=33.8 K 2024-11-19T08:41:40,794 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/96ee329ddae74681845392502d171a72, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/2feca0e9564f4f70a7ab61abe9d4940f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/29aca2cb06bc43a986490295d34690fd] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=91.6 K 2024-11-19T08:41:40,794 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:40,794 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/96ee329ddae74681845392502d171a72, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/2feca0e9564f4f70a7ab61abe9d4940f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/29aca2cb06bc43a986490295d34690fd] 2024-11-19T08:41:40,794 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc20e971ce84837813de313f0ac283e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732005695603 2024-11-19T08:41:40,795 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca63d65a73447bfb88f2b7a864364e8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732005696794 2024-11-19T08:41:40,795 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96ee329ddae74681845392502d171a72, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732005695603 2024-11-19T08:41:40,795 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 00aa6d4828644246bd4f4c530567bc60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732005698983 2024-11-19T08:41:40,795 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2feca0e9564f4f70a7ab61abe9d4940f, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732005696794 2024-11-19T08:41:40,795 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29aca2cb06bc43a986490295d34690fd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732005698983 2024-11-19T08:41:40,802 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:40,802 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/c66e088690624a0d9e05f43ac0f78526 is 50, key is test_row_0/B:col10/1732005698983/Put/seqid=0 2024-11-19T08:41:40,803 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:40,816 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111992601b8133ee4eafae3838028e28dda1_8f3ff379c750bd3082218a4d8d91a02f store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:40,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742207_1383 (size=12765) 2024-11-19T08:41:40,819 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111992601b8133ee4eafae3838028e28dda1_8f3ff379c750bd3082218a4d8d91a02f, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:40,819 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111992601b8133ee4eafae3838028e28dda1_8f3ff379c750bd3082218a4d8d91a02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:40,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742208_1384 (size=4469) 2024-11-19T08:41:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T08:41:40,936 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:40,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T08:41:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T08:41:40,938 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:40,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:40,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T08:41:41,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T08:41:41,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:41,091 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:41:41,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191aa9e2c20f044214b245c9c537820dd7_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005699241/Put/seqid=0 2024-11-19T08:41:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742209_1385 (size=12454) 2024-11-19T08:41:41,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:41,105 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191aa9e2c20f044214b245c9c537820dd7_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191aa9e2c20f044214b245c9c537820dd7_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:41,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/dbf245ead05d4f0b83e5cf2a22633929, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/dbf245ead05d4f0b83e5cf2a22633929 is 175, key is test_row_0/A:col10/1732005699241/Put/seqid=0 2024-11-19T08:41:41,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742210_1386 (size=31255) 2024-11-19T08:41:41,116 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/dbf245ead05d4f0b83e5cf2a22633929 2024-11-19T08:41:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/4ebb761f94864eb093fab0e471167494 is 50, key is test_row_0/B:col10/1732005699241/Put/seqid=0 2024-11-19T08:41:41,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742211_1387 (size=12301) 2024-11-19T08:41:41,221 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/c66e088690624a0d9e05f43ac0f78526 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/c66e088690624a0d9e05f43ac0f78526 2024-11-19T08:41:41,225 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/B of 8f3ff379c750bd3082218a4d8d91a02f into c66e088690624a0d9e05f43ac0f78526(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:41,225 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:41,225 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/B, priority=13, startTime=1732005700793; duration=0sec 2024-11-19T08:41:41,225 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:41,225 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:B 2024-11-19T08:41:41,225 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:41,225 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:41,226 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:41,226 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/C is initiating minor compaction (all files) 2024-11-19T08:41:41,226 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/C in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:41,226 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/cec5560570f84b1c9b9635a25157f005 is 175, key is test_row_0/A:col10/1732005698983/Put/seqid=0 2024-11-19T08:41:41,226 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/7033c0d8fb994a06977d2652a6acaa00, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/620baace61e84b588ff9cf48c9973c95, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/c8128ae53dd646d98d402e43e6010ed7] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=33.8 K 2024-11-19T08:41:41,226 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7033c0d8fb994a06977d2652a6acaa00, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732005695603 2024-11-19T08:41:41,226 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 620baace61e84b588ff9cf48c9973c95, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732005696794 2024-11-19T08:41:41,227 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c8128ae53dd646d98d402e43e6010ed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732005698983 2024-11-19T08:41:41,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742212_1388 (size=31719) 2024-11-19T08:41:41,237 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#C#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:41,237 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/cdb6f64be9d14c93a19d71d0e7b7e52a is 50, key is test_row_0/C:col10/1732005698983/Put/seqid=0 2024-11-19T08:41:41,243 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/cec5560570f84b1c9b9635a25157f005 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/cec5560570f84b1c9b9635a25157f005 2024-11-19T08:41:41,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742213_1389 (size=12765) 2024-11-19T08:41:41,250 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/A of 8f3ff379c750bd3082218a4d8d91a02f into cec5560570f84b1c9b9635a25157f005(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:41,250 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:41,250 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/A, priority=13, startTime=1732005700793; duration=0sec 2024-11-19T08:41:41,250 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:41,250 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:A 2024-11-19T08:41:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T08:41:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:41,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:41,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005761445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:41,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005761448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:41,531 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/4ebb761f94864eb093fab0e471167494 2024-11-19T08:41:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005761556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005761556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T08:41:41,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/de09dfbaf3cc4d6985e0683c72c591c2 is 50, key is test_row_0/C:col10/1732005699241/Put/seqid=0 2024-11-19T08:41:41,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742214_1390 (size=12301) 2024-11-19T08:41:41,648 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/cdb6f64be9d14c93a19d71d0e7b7e52a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdb6f64be9d14c93a19d71d0e7b7e52a 2024-11-19T08:41:41,653 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/C of 8f3ff379c750bd3082218a4d8d91a02f into cdb6f64be9d14c93a19d71d0e7b7e52a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:41,654 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:41,654 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/C, priority=13, startTime=1732005700793; duration=0sec 2024-11-19T08:41:41,654 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:41,654 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:C 2024-11-19T08:41:41,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005761765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:41,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005761766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:41,983 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/de09dfbaf3cc4d6985e0683c72c591c2 2024-11-19T08:41:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/dbf245ead05d4f0b83e5cf2a22633929 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/dbf245ead05d4f0b83e5cf2a22633929 2024-11-19T08:41:41,993 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/dbf245ead05d4f0b83e5cf2a22633929, entries=150, sequenceid=280, filesize=30.5 K 2024-11-19T08:41:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/4ebb761f94864eb093fab0e471167494 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4ebb761f94864eb093fab0e471167494 2024-11-19T08:41:41,997 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4ebb761f94864eb093fab0e471167494, entries=150, sequenceid=280, filesize=12.0 K 2024-11-19T08:41:41,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/de09dfbaf3cc4d6985e0683c72c591c2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/de09dfbaf3cc4d6985e0683c72c591c2 2024-11-19T08:41:42,001 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/de09dfbaf3cc4d6985e0683c72c591c2, entries=150, sequenceid=280, filesize=12.0 K 2024-11-19T08:41:42,001 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8f3ff379c750bd3082218a4d8d91a02f in 910ms, sequenceid=280, compaction requested=false 2024-11-19T08:41:42,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:42,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T08:41:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T08:41:42,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T08:41:42,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0640 sec 2024-11-19T08:41:42,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0690 sec 2024-11-19T08:41:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T08:41:42,076 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:42,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T08:41:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T08:41:42,078 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:42,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:42,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:42,081 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:42,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:41:42,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119021fbc256b3f4c07a7d711e6ec90bbcd_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005701437/Put/seqid=0 2024-11-19T08:41:42,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742215_1391 (size=14994) 2024-11-19T08:41:42,099 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:42,102 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119021fbc256b3f4c07a7d711e6ec90bbcd_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119021fbc256b3f4c07a7d711e6ec90bbcd_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:42,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d8d7e63d02e1416391bc3af5f0f44ad0, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:42,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d8d7e63d02e1416391bc3af5f0f44ad0 is 175, key is test_row_0/A:col10/1732005701437/Put/seqid=0 2024-11-19T08:41:42,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742216_1392 (size=39949) 2024-11-19T08:41:42,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005762156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005762161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T08:41:42,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T08:41:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005762276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005762276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T08:41:42,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T08:41:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005762486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005762486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,507 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d8d7e63d02e1416391bc3af5f0f44ad0 2024-11-19T08:41:42,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/72864f1493c14b9186174d9657ec6f53 is 50, key is test_row_0/B:col10/1732005701437/Put/seqid=0 2024-11-19T08:41:42,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742217_1393 (size=12301) 2024-11-19T08:41:42,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/72864f1493c14b9186174d9657ec6f53 2024-11-19T08:41:42,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/183a4c18021f4bcf8427cdd39a6e5b03 is 50, key is test_row_0/C:col10/1732005701437/Put/seqid=0 2024-11-19T08:41:42,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742218_1394 (size=12301) 2024-11-19T08:41:42,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T08:41:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:42,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T08:41:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T08:41:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43302 deadline: 1732005762805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43298 deadline: 1732005762806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:41:42,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T08:41:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:42,884 INFO [Thread-1434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1434. 2024-11-19T08:41:42,885 DEBUG [Thread-1434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:42,885 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,885 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:42,891 INFO [Thread-1442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1442. 2024-11-19T08:41:42,891 INFO [Thread-1440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1440. 2024-11-19T08:41:42,891 DEBUG [Thread-1442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:42,891 DEBUG [Thread-1440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:42,891 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,891 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,891 DEBUG [Thread-1440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:42,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:42,891 DEBUG [Thread-1442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,894 INFO [Thread-1438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1438. 2024-11-19T08:41:42,894 DEBUG [Thread-1438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:42,894 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,894 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:42,900 INFO [Thread-1436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1436. 2024-11-19T08:41:42,900 DEBUG [Thread-1436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:42,900 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,900 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:42,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:42,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/183a4c18021f4bcf8427cdd39a6e5b03 2024-11-19T08:41:42,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/d8d7e63d02e1416391bc3af5f0f44ad0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d8d7e63d02e1416391bc3af5f0f44ad0 2024-11-19T08:41:42,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d8d7e63d02e1416391bc3af5f0f44ad0, entries=200, sequenceid=296, filesize=39.0 K 2024-11-19T08:41:42,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/72864f1493c14b9186174d9657ec6f53 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/72864f1493c14b9186174d9657ec6f53 2024-11-19T08:41:42,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/72864f1493c14b9186174d9657ec6f53, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T08:41:42,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/183a4c18021f4bcf8427cdd39a6e5b03 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/183a4c18021f4bcf8427cdd39a6e5b03 2024-11-19T08:41:42,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/183a4c18021f4bcf8427cdd39a6e5b03, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T08:41:42,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8f3ff379c750bd3082218a4d8d91a02f in 876ms, sequenceid=296, compaction requested=true 2024-11-19T08:41:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:42,957 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:42,957 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3ff379c750bd3082218a4d8d91a02f:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:42,959 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:42,959 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:42,959 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/A is initiating minor compaction (all files) 2024-11-19T08:41:42,959 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/B is initiating minor compaction (all files) 2024-11-19T08:41:42,959 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/A in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,959 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/B in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,959 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/cec5560570f84b1c9b9635a25157f005, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/dbf245ead05d4f0b83e5cf2a22633929, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d8d7e63d02e1416391bc3af5f0f44ad0] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=100.5 K 2024-11-19T08:41:42,959 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/c66e088690624a0d9e05f43ac0f78526, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4ebb761f94864eb093fab0e471167494, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/72864f1493c14b9186174d9657ec6f53] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=36.5 K 2024-11-19T08:41:42,959 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,959 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/cec5560570f84b1c9b9635a25157f005, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/dbf245ead05d4f0b83e5cf2a22633929, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d8d7e63d02e1416391bc3af5f0f44ad0] 2024-11-19T08:41:42,959 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c66e088690624a0d9e05f43ac0f78526, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732005698983 2024-11-19T08:41:42,959 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting cec5560570f84b1c9b9635a25157f005, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732005698983 2024-11-19T08:41:42,960 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ebb761f94864eb093fab0e471167494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732005699228 2024-11-19T08:41:42,960 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf245ead05d4f0b83e5cf2a22633929, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732005699228 2024-11-19T08:41:42,960 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72864f1493c14b9186174d9657ec6f53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732005701437 2024-11-19T08:41:42,960 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d7e63d02e1416391bc3af5f0f44ad0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732005701437 2024-11-19T08:41:42,969 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:42,969 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:42,970 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/bfcf5351f7ca410780eaea33877fa401 is 50, key is test_row_0/B:col10/1732005701437/Put/seqid=0 2024-11-19T08:41:42,971 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119ab7fe0e8819d4d9397cd148a92747217_8f3ff379c750bd3082218a4d8d91a02f store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:42,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742219_1395 (size=13017) 2024-11-19T08:41:42,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T08:41:42,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:42,995 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:41:42,995 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119ab7fe0e8819d4d9397cd148a92747217_8f3ff379c750bd3082218a4d8d91a02f, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:42,995 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ab7fe0e8819d4d9397cd148a92747217_8f3ff379c750bd3082218a4d8d91a02f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:42,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742220_1396 (size=4469) 2024-11-19T08:41:43,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111932d6ce1dd5394d02af52bacdd6acf29c_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005702154/Put/seqid=0 2024-11-19T08:41:43,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742221_1397 (size=12454) 2024-11-19T08:41:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T08:41:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:43,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. as already flushing 2024-11-19T08:41:43,330 INFO [Thread-1425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1425. 2024-11-19T08:41:43,330 INFO [Thread-1427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1427. 2024-11-19T08:41:43,330 DEBUG [Thread-1425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:43,330 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,330 DEBUG [Thread-1427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:43,330 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,330 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:43,330 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:43,385 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/bfcf5351f7ca410780eaea33877fa401 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/bfcf5351f7ca410780eaea33877fa401 2024-11-19T08:41:43,391 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/B of 8f3ff379c750bd3082218a4d8d91a02f into bfcf5351f7ca410780eaea33877fa401(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:43,391 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:43,391 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/B, priority=13, startTime=1732005702957; duration=0sec 2024-11-19T08:41:43,391 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:43,391 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:B 2024-11-19T08:41:43,391 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:43,392 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:43,392 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): 8f3ff379c750bd3082218a4d8d91a02f/C is initiating minor compaction (all files) 2024-11-19T08:41:43,392 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3ff379c750bd3082218a4d8d91a02f/C in TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:43,392 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdb6f64be9d14c93a19d71d0e7b7e52a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/de09dfbaf3cc4d6985e0683c72c591c2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/183a4c18021f4bcf8427cdd39a6e5b03] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp, totalSize=36.5 K 2024-11-19T08:41:43,393 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb6f64be9d14c93a19d71d0e7b7e52a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732005698983 2024-11-19T08:41:43,393 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting de09dfbaf3cc4d6985e0683c72c591c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732005699228 2024-11-19T08:41:43,394 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 183a4c18021f4bcf8427cdd39a6e5b03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732005701437 2024-11-19T08:41:43,400 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#A#compaction#331 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:43,401 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/ac58bdcdf5054bfcb6205473845363b1 is 175, key is test_row_0/A:col10/1732005701437/Put/seqid=0 2024-11-19T08:41:43,404 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3ff379c750bd3082218a4d8d91a02f#C#compaction#333 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:43,404 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/f410764447aa4ca38b8705dc4be1c60d is 50, key is test_row_0/C:col10/1732005701437/Put/seqid=0 2024-11-19T08:41:43,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742222_1398 (size=31971) 2024-11-19T08:41:43,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742223_1399 (size=13017) 2024-11-19T08:41:43,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:43,414 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111932d6ce1dd5394d02af52bacdd6acf29c_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111932d6ce1dd5394d02af52bacdd6acf29c_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:43,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/1b758bc5bc864fe4b04822fa02f85595, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:43,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/1b758bc5bc864fe4b04822fa02f85595 is 175, key is test_row_0/A:col10/1732005702154/Put/seqid=0 2024-11-19T08:41:43,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742224_1400 (size=31255) 2024-11-19T08:41:43,611 INFO [Thread-1429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1429. 2024-11-19T08:41:43,612 DEBUG [Thread-1429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:43,612 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,612 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:43,631 INFO [Thread-1423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1423. 2024-11-19T08:41:43,631 INFO [Thread-1431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1431. 2024-11-19T08:41:43,631 DEBUG [Thread-1423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:43,631 DEBUG [Thread-1431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:41:43,631 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,631 DEBUG [Thread-1431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,631 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,632 DEBUG [Thread-1431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:43,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:43,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:43,813 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/ac58bdcdf5054bfcb6205473845363b1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ac58bdcdf5054bfcb6205473845363b1 2024-11-19T08:41:43,816 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/f410764447aa4ca38b8705dc4be1c60d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f410764447aa4ca38b8705dc4be1c60d 2024-11-19T08:41:43,820 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/A of 8f3ff379c750bd3082218a4d8d91a02f into ac58bdcdf5054bfcb6205473845363b1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:43,820 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:43,820 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/1b758bc5bc864fe4b04822fa02f85595 2024-11-19T08:41:43,820 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/A, priority=13, startTime=1732005702957; duration=0sec 2024-11-19T08:41:43,820 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:43,820 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:A 2024-11-19T08:41:43,821 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3ff379c750bd3082218a4d8d91a02f/C of 8f3ff379c750bd3082218a4d8d91a02f into f410764447aa4ca38b8705dc4be1c60d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:43,821 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:43,821 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., storeName=8f3ff379c750bd3082218a4d8d91a02f/C, priority=13, startTime=1732005702957; duration=0sec 2024-11-19T08:41:43,821 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:43,821 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3ff379c750bd3082218a4d8d91a02f:C 2024-11-19T08:41:43,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/97f88ad79ec644c9a8c7784050bf7af4 is 50, key is test_row_0/B:col10/1732005702154/Put/seqid=0 2024-11-19T08:41:43,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742225_1401 (size=12301) 2024-11-19T08:41:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T08:41:44,230 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/97f88ad79ec644c9a8c7784050bf7af4 2024-11-19T08:41:44,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/cdc42a20b73641c284290cf4ed10fa8f is 50, key is test_row_0/C:col10/1732005702154/Put/seqid=0 2024-11-19T08:41:44,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742226_1402 (size=12301) 2024-11-19T08:41:44,645 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/cdc42a20b73641c284290cf4ed10fa8f 2024-11-19T08:41:44,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/1b758bc5bc864fe4b04822fa02f85595 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/1b758bc5bc864fe4b04822fa02f85595 2024-11-19T08:41:44,661 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/1b758bc5bc864fe4b04822fa02f85595, entries=150, sequenceid=319, filesize=30.5 K 2024-11-19T08:41:44,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/97f88ad79ec644c9a8c7784050bf7af4 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/97f88ad79ec644c9a8c7784050bf7af4 2024-11-19T08:41:44,665 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/97f88ad79ec644c9a8c7784050bf7af4, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T08:41:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/cdc42a20b73641c284290cf4ed10fa8f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdc42a20b73641c284290cf4ed10fa8f 2024-11-19T08:41:44,669 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdc42a20b73641c284290cf4ed10fa8f, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T08:41:44,670 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=33.54 KB/34350 for 8f3ff379c750bd3082218a4d8d91a02f in 1675ms, sequenceid=319, compaction requested=false 2024-11-19T08:41:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 8f3ff379c750bd3082218a4d8d91a02f: 2024-11-19T08:41:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T08:41:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T08:41:44,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T08:41:44,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5930 sec 2024-11-19T08:41:44,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5970 sec 2024-11-19T08:41:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T08:41:46,238 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1345 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4027 rows 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1344 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4024 rows 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1352 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4056 rows 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1351 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4048 rows 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1356 2024-11-19T08:41:46,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4059 rows 2024-11-19T08:41:46,239 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:41:46,239 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:41:46,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:46,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:46,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:41:46,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T08:41:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T08:41:46,245 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005706245"}]},"ts":"1732005706245"} 2024-11-19T08:41:46,246 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T08:41:46,246 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T08:41:46,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:41:46,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, UNASSIGN}] 2024-11-19T08:41:46,250 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, UNASSIGN 2024-11-19T08:41:46,250 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:46,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, UNASSIGN because future has completed 2024-11-19T08:41:46,252 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:41:46,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:41:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T08:41:46,405 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:46,405 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:41:46,405 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 8f3ff379c750bd3082218a4d8d91a02f, disabling compactions & flushes 2024-11-19T08:41:46,405 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:46,405 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:46,405 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. after waiting 0 ms 2024-11-19T08:41:46,405 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:46,405 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 8f3ff379c750bd3082218a4d8d91a02f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T08:41:46,416 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111965b7f9a19ae04a1d8cae9edc52277359_8f3ff379c750bd3082218a4d8d91a02f is 50, key is test_row_0/A:col10/1732005703628/Put/seqid=0 2024-11-19T08:41:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742227_1403 (size=12454) 2024-11-19T08:41:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T08:41:46,821 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T08:41:46,869 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111965b7f9a19ae04a1d8cae9edc52277359_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965b7f9a19ae04a1d8cae9edc52277359_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:46,871 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/647122f404b340929ec24b9f9f1d8c45, store: [table=TestAcidGuarantees family=A region=8f3ff379c750bd3082218a4d8d91a02f] 2024-11-19T08:41:46,872 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/647122f404b340929ec24b9f9f1d8c45 is 175, key is test_row_0/A:col10/1732005703628/Put/seqid=0 2024-11-19T08:41:46,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742228_1404 (size=31255) 2024-11-19T08:41:47,278 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/647122f404b340929ec24b9f9f1d8c45 2024-11-19T08:41:47,291 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/cd666db29aa64aa28fa9e7555faedaf0 is 50, key is test_row_0/B:col10/1732005703628/Put/seqid=0 2024-11-19T08:41:47,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742229_1405 (size=12301) 2024-11-19T08:41:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T08:41:47,697 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/cd666db29aa64aa28fa9e7555faedaf0 2024-11-19T08:41:47,709 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/963c5baef89e470abfebd67d064290fd is 50, key is test_row_0/C:col10/1732005703628/Put/seqid=0 2024-11-19T08:41:47,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742230_1406 (size=12301) 2024-11-19T08:41:48,113 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/963c5baef89e470abfebd67d064290fd 2024-11-19T08:41:48,121 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/A/647122f404b340929ec24b9f9f1d8c45 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/647122f404b340929ec24b9f9f1d8c45 2024-11-19T08:41:48,127 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/647122f404b340929ec24b9f9f1d8c45, entries=150, sequenceid=330, filesize=30.5 K 2024-11-19T08:41:48,128 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/B/cd666db29aa64aa28fa9e7555faedaf0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd666db29aa64aa28fa9e7555faedaf0 2024-11-19T08:41:48,132 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd666db29aa64aa28fa9e7555faedaf0, entries=150, sequenceid=330, filesize=12.0 K 2024-11-19T08:41:48,133 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/.tmp/C/963c5baef89e470abfebd67d064290fd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/963c5baef89e470abfebd67d064290fd 2024-11-19T08:41:48,137 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/963c5baef89e470abfebd67d064290fd, entries=150, sequenceid=330, filesize=12.0 K 2024-11-19T08:41:48,137 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8f3ff379c750bd3082218a4d8d91a02f in 1732ms, sequenceid=330, compaction requested=true 2024-11-19T08:41:48,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/83875ee1b05d4313b6dcdc784698dc1b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/0273427c945849748edc154067de328f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/63f88188936f40afac3f703bf3af78fd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d9263abd670c486599b977415811ea1f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ffcde04c390f451193f39907f4a972ff, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/fed1991880224f059f08000320cbb2a9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c9f002af90c14a84b91d5f9f0e94cdbe, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c387818ab1ef488c815ec5dd97ffdb64, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/b2a048d9811b40c88588d908ee52037a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/f5fb24cea9cc43eda6b20918470f7ae1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a557d7bf4aad4af48d2b76f5e2cb492c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a08dceb0d5e0482c89e442ff899cc67d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d2b0bc7c1b804a75b50a930686c10717, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/e7a4ecb1bcc045cc9e18215606ae4347, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/37c888a81da34d0aa88c7afd0388910b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/96ee329ddae74681845392502d171a72, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/2feca0e9564f4f70a7ab61abe9d4940f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/29aca2cb06bc43a986490295d34690fd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/cec5560570f84b1c9b9635a25157f005, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/dbf245ead05d4f0b83e5cf2a22633929, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d8d7e63d02e1416391bc3af5f0f44ad0] to archive 2024-11-19T08:41:48,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:48,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/83875ee1b05d4313b6dcdc784698dc1b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/83875ee1b05d4313b6dcdc784698dc1b 2024-11-19T08:41:48,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/0273427c945849748edc154067de328f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/0273427c945849748edc154067de328f 2024-11-19T08:41:48,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/63f88188936f40afac3f703bf3af78fd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/63f88188936f40afac3f703bf3af78fd 2024-11-19T08:41:48,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d9263abd670c486599b977415811ea1f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d9263abd670c486599b977415811ea1f 2024-11-19T08:41:48,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ffcde04c390f451193f39907f4a972ff to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ffcde04c390f451193f39907f4a972ff 2024-11-19T08:41:48,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/fed1991880224f059f08000320cbb2a9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/fed1991880224f059f08000320cbb2a9 2024-11-19T08:41:48,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c9f002af90c14a84b91d5f9f0e94cdbe to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c9f002af90c14a84b91d5f9f0e94cdbe 2024-11-19T08:41:48,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c387818ab1ef488c815ec5dd97ffdb64 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/c387818ab1ef488c815ec5dd97ffdb64 2024-11-19T08:41:48,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/b2a048d9811b40c88588d908ee52037a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/b2a048d9811b40c88588d908ee52037a 2024-11-19T08:41:48,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/f5fb24cea9cc43eda6b20918470f7ae1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/f5fb24cea9cc43eda6b20918470f7ae1 2024-11-19T08:41:48,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a557d7bf4aad4af48d2b76f5e2cb492c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a557d7bf4aad4af48d2b76f5e2cb492c 2024-11-19T08:41:48,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a08dceb0d5e0482c89e442ff899cc67d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/a08dceb0d5e0482c89e442ff899cc67d 2024-11-19T08:41:48,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d2b0bc7c1b804a75b50a930686c10717 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d2b0bc7c1b804a75b50a930686c10717 2024-11-19T08:41:48,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/e7a4ecb1bcc045cc9e18215606ae4347 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/e7a4ecb1bcc045cc9e18215606ae4347 2024-11-19T08:41:48,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/37c888a81da34d0aa88c7afd0388910b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/37c888a81da34d0aa88c7afd0388910b 2024-11-19T08:41:48,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/96ee329ddae74681845392502d171a72 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/96ee329ddae74681845392502d171a72 2024-11-19T08:41:48,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/2feca0e9564f4f70a7ab61abe9d4940f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/2feca0e9564f4f70a7ab61abe9d4940f 2024-11-19T08:41:48,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/29aca2cb06bc43a986490295d34690fd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/29aca2cb06bc43a986490295d34690fd 2024-11-19T08:41:48,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/cec5560570f84b1c9b9635a25157f005 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/cec5560570f84b1c9b9635a25157f005 2024-11-19T08:41:48,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/dbf245ead05d4f0b83e5cf2a22633929 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/dbf245ead05d4f0b83e5cf2a22633929 2024-11-19T08:41:48,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d8d7e63d02e1416391bc3af5f0f44ad0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/d8d7e63d02e1416391bc3af5f0f44ad0 2024-11-19T08:41:48,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/98c9994c8d754046bdf4ed8f1f718b9b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/749fadc57aa34916a06f9a149ac10375, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd51077ac2c24fbd9b689b66c1a046a3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/99d2c4dd905e40e1b3a1aeaea8660f43, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/30f831dbedb243d2b00904fd09b26437, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/b889d027dfc3489c8a1d61d6ba6dfe29, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/54a7e05f5b85424c8196623433f77e5e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8360533e3644469e8444e17dd85971a8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/a22f153dc66b4df58c336cd834a6308a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/3882d1290fa54e5caefff7e3a29f852a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/f29df51172b64b22867e2bed5a8d974e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8a7d5df5759c48978e025ec439fb0467, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/126f60d50af446efa760dbe11f18f320, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/26ccbd28e3624cd680af037eeffd678c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4cc20e971ce84837813de313f0ac283e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/be8adbe8042e45a39655a9811de90103, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/2ca63d65a73447bfb88f2b7a864364e8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/c66e088690624a0d9e05f43ac0f78526, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/00aa6d4828644246bd4f4c530567bc60, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4ebb761f94864eb093fab0e471167494, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/72864f1493c14b9186174d9657ec6f53] to archive 2024-11-19T08:41:48,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:48,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/98c9994c8d754046bdf4ed8f1f718b9b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/98c9994c8d754046bdf4ed8f1f718b9b 2024-11-19T08:41:48,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/749fadc57aa34916a06f9a149ac10375 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/749fadc57aa34916a06f9a149ac10375 2024-11-19T08:41:48,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd51077ac2c24fbd9b689b66c1a046a3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd51077ac2c24fbd9b689b66c1a046a3 2024-11-19T08:41:48,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/99d2c4dd905e40e1b3a1aeaea8660f43 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/99d2c4dd905e40e1b3a1aeaea8660f43 2024-11-19T08:41:48,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/30f831dbedb243d2b00904fd09b26437 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/30f831dbedb243d2b00904fd09b26437 2024-11-19T08:41:48,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/b889d027dfc3489c8a1d61d6ba6dfe29 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/b889d027dfc3489c8a1d61d6ba6dfe29 2024-11-19T08:41:48,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/54a7e05f5b85424c8196623433f77e5e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/54a7e05f5b85424c8196623433f77e5e 2024-11-19T08:41:48,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8360533e3644469e8444e17dd85971a8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8360533e3644469e8444e17dd85971a8 2024-11-19T08:41:48,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/a22f153dc66b4df58c336cd834a6308a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/a22f153dc66b4df58c336cd834a6308a 2024-11-19T08:41:48,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/3882d1290fa54e5caefff7e3a29f852a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/3882d1290fa54e5caefff7e3a29f852a 2024-11-19T08:41:48,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/f29df51172b64b22867e2bed5a8d974e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/f29df51172b64b22867e2bed5a8d974e 2024-11-19T08:41:48,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8a7d5df5759c48978e025ec439fb0467 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/8a7d5df5759c48978e025ec439fb0467 2024-11-19T08:41:48,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/126f60d50af446efa760dbe11f18f320 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/126f60d50af446efa760dbe11f18f320 2024-11-19T08:41:48,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/26ccbd28e3624cd680af037eeffd678c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/26ccbd28e3624cd680af037eeffd678c 2024-11-19T08:41:48,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4cc20e971ce84837813de313f0ac283e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4cc20e971ce84837813de313f0ac283e 2024-11-19T08:41:48,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/be8adbe8042e45a39655a9811de90103 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/be8adbe8042e45a39655a9811de90103 2024-11-19T08:41:48,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/2ca63d65a73447bfb88f2b7a864364e8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/2ca63d65a73447bfb88f2b7a864364e8 2024-11-19T08:41:48,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/c66e088690624a0d9e05f43ac0f78526 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/c66e088690624a0d9e05f43ac0f78526 2024-11-19T08:41:48,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/00aa6d4828644246bd4f4c530567bc60 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/00aa6d4828644246bd4f4c530567bc60 2024-11-19T08:41:48,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4ebb761f94864eb093fab0e471167494 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/4ebb761f94864eb093fab0e471167494 2024-11-19T08:41:48,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/72864f1493c14b9186174d9657ec6f53 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/72864f1493c14b9186174d9657ec6f53 2024-11-19T08:41:48,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b87d2ea737074a67bd3a47c298241734, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f2dff8a289b847f6af1ee51d89ea69f0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b9e2c5c629a94313a977822ce86ea423, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/33436b69b77b440a90f05ed66d308772, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/20234862f41a4430a2d3b936abb95bc1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/6132dd68c9e8442882b7e1243b93afa2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/045ed02a3f664e28b87b64d58978aa37, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/345f1fe0e51743ed8b2d3fcffd25fc3b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/91d78a4a7475478d859f472c8c5b1c5f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/54ac0126982d484f818838c884339d46, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/67c33f7a6411487fa181cb15db06d306, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/3b93d03fe0944eccbb1ee375b801a9e9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/0ca3ad977e9b434ca3d3df02730e68ca, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/89f0fd68d6c9416fbd6049a11bd9467b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/7033c0d8fb994a06977d2652a6acaa00, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/74b3cf3390e64b66af85f4b43f5b1d4f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/620baace61e84b588ff9cf48c9973c95, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdb6f64be9d14c93a19d71d0e7b7e52a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/c8128ae53dd646d98d402e43e6010ed7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/de09dfbaf3cc4d6985e0683c72c591c2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/183a4c18021f4bcf8427cdd39a6e5b03] to archive 2024-11-19T08:41:48,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:48,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b87d2ea737074a67bd3a47c298241734 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b87d2ea737074a67bd3a47c298241734 2024-11-19T08:41:48,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f2dff8a289b847f6af1ee51d89ea69f0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f2dff8a289b847f6af1ee51d89ea69f0 2024-11-19T08:41:48,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b9e2c5c629a94313a977822ce86ea423 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/b9e2c5c629a94313a977822ce86ea423 2024-11-19T08:41:48,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/33436b69b77b440a90f05ed66d308772 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/33436b69b77b440a90f05ed66d308772 2024-11-19T08:41:48,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/20234862f41a4430a2d3b936abb95bc1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/20234862f41a4430a2d3b936abb95bc1 2024-11-19T08:41:48,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/6132dd68c9e8442882b7e1243b93afa2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/6132dd68c9e8442882b7e1243b93afa2 2024-11-19T08:41:48,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/045ed02a3f664e28b87b64d58978aa37 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/045ed02a3f664e28b87b64d58978aa37 2024-11-19T08:41:48,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/345f1fe0e51743ed8b2d3fcffd25fc3b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/345f1fe0e51743ed8b2d3fcffd25fc3b 2024-11-19T08:41:48,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/91d78a4a7475478d859f472c8c5b1c5f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/91d78a4a7475478d859f472c8c5b1c5f 2024-11-19T08:41:48,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/54ac0126982d484f818838c884339d46 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/54ac0126982d484f818838c884339d46 2024-11-19T08:41:48,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/67c33f7a6411487fa181cb15db06d306 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/67c33f7a6411487fa181cb15db06d306 2024-11-19T08:41:48,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/3b93d03fe0944eccbb1ee375b801a9e9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/3b93d03fe0944eccbb1ee375b801a9e9 2024-11-19T08:41:48,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/0ca3ad977e9b434ca3d3df02730e68ca to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/0ca3ad977e9b434ca3d3df02730e68ca 2024-11-19T08:41:48,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/89f0fd68d6c9416fbd6049a11bd9467b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/89f0fd68d6c9416fbd6049a11bd9467b 2024-11-19T08:41:48,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/7033c0d8fb994a06977d2652a6acaa00 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/7033c0d8fb994a06977d2652a6acaa00 2024-11-19T08:41:48,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/74b3cf3390e64b66af85f4b43f5b1d4f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/74b3cf3390e64b66af85f4b43f5b1d4f 2024-11-19T08:41:48,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/620baace61e84b588ff9cf48c9973c95 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/620baace61e84b588ff9cf48c9973c95 2024-11-19T08:41:48,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdb6f64be9d14c93a19d71d0e7b7e52a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdb6f64be9d14c93a19d71d0e7b7e52a 2024-11-19T08:41:48,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/c8128ae53dd646d98d402e43e6010ed7 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/c8128ae53dd646d98d402e43e6010ed7 2024-11-19T08:41:48,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/de09dfbaf3cc4d6985e0683c72c591c2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/de09dfbaf3cc4d6985e0683c72c591c2 2024-11-19T08:41:48,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/183a4c18021f4bcf8427cdd39a6e5b03 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/183a4c18021f4bcf8427cdd39a6e5b03 2024-11-19T08:41:48,221 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/recovered.edits/333.seqid, newMaxSeqId=333, maxSeqId=4 2024-11-19T08:41:48,222 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. 2024-11-19T08:41:48,222 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 8f3ff379c750bd3082218a4d8d91a02f: Waiting for close lock at 1732005706405Running coprocessor pre-close hooks at 1732005706405Disabling compacts and flushes for region at 1732005706405Disabling writes for close at 1732005706405Obtaining lock to block concurrent updates at 1732005706405Preparing flush snapshotting stores in 8f3ff379c750bd3082218a4d8d91a02f at 1732005706405Finished memstore snapshotting TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732005706406 (+1 ms)Flushing stores of TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f. at 1732005706406Flushing 8f3ff379c750bd3082218a4d8d91a02f/A: creating writer at 1732005706406Flushing mob file 8f3ff379c750bd3082218a4d8d91a02f/A: appending metadata at 1732005706416 (+10 ms)Flushing mob file 8f3ff379c750bd3082218a4d8d91a02f/A: closing flushed file at 1732005706416Flushing 8f3ff379c750bd3082218a4d8d91a02f/A: appending metadata at 1732005706871 (+455 ms)Flushing 8f3ff379c750bd3082218a4d8d91a02f/A: closing flushed file at 1732005706871Flushing 8f3ff379c750bd3082218a4d8d91a02f/B: creating writer at 1732005707286 (+415 ms)Flushing 8f3ff379c750bd3082218a4d8d91a02f/B: appending metadata at 1732005707291 (+5 ms)Flushing 8f3ff379c750bd3082218a4d8d91a02f/B: closing flushed file at 1732005707291Flushing 8f3ff379c750bd3082218a4d8d91a02f/C: creating writer at 1732005707705 (+414 ms)Flushing 8f3ff379c750bd3082218a4d8d91a02f/C: appending metadata at 1732005707709 (+4 ms)Flushing 8f3ff379c750bd3082218a4d8d91a02f/C: closing flushed file at 1732005707709Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b4db857: reopening flushed file at 1732005708120 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cb1d1e8: reopening flushed file at 1732005708127 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3da818dc: reopening flushed file at 1732005708132 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8f3ff379c750bd3082218a4d8d91a02f in 1732ms, sequenceid=330, compaction requested=true at 1732005708137 (+5 ms)Writing region close event to WAL at 1732005708218 (+81 ms)Running coprocessor post-close hooks at 1732005708222 (+4 ms)Closed at 1732005708222 2024-11-19T08:41:48,224 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=8f3ff379c750bd3082218a4d8d91a02f, regionState=CLOSED 2024-11-19T08:41:48,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:41:48,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T08:41:48,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f3ff379c750bd3082218a4d8d91a02f, server=3ab37fa97a98,34165,1732005593269 in 1.9750 sec 2024-11-19T08:41:48,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-19T08:41:48,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3ff379c750bd3082218a4d8d91a02f, UNASSIGN in 1.9800 sec 2024-11-19T08:41:48,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T08:41:48,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9850 sec 2024-11-19T08:41:48,235 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005708235"}]},"ts":"1732005708235"} 2024-11-19T08:41:48,237 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T08:41:48,237 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T08:41:48,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9980 sec 2024-11-19T08:41:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T08:41:48,387 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:48,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T08:41:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:48,393 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T08:41:48,394 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:48,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,404 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/recovered.edits] 2024-11-19T08:41:48,408 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/1b758bc5bc864fe4b04822fa02f85595 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/1b758bc5bc864fe4b04822fa02f85595 2024-11-19T08:41:48,409 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/647122f404b340929ec24b9f9f1d8c45 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/647122f404b340929ec24b9f9f1d8c45 2024-11-19T08:41:48,411 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ac58bdcdf5054bfcb6205473845363b1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/A/ac58bdcdf5054bfcb6205473845363b1 2024-11-19T08:41:48,414 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/97f88ad79ec644c9a8c7784050bf7af4 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/97f88ad79ec644c9a8c7784050bf7af4 2024-11-19T08:41:48,416 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/bfcf5351f7ca410780eaea33877fa401 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/bfcf5351f7ca410780eaea33877fa401 2024-11-19T08:41:48,418 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd666db29aa64aa28fa9e7555faedaf0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/B/cd666db29aa64aa28fa9e7555faedaf0 2024-11-19T08:41:48,421 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/963c5baef89e470abfebd67d064290fd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/963c5baef89e470abfebd67d064290fd 2024-11-19T08:41:48,423 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdc42a20b73641c284290cf4ed10fa8f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/cdc42a20b73641c284290cf4ed10fa8f 2024-11-19T08:41:48,425 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f410764447aa4ca38b8705dc4be1c60d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/C/f410764447aa4ca38b8705dc4be1c60d 2024-11-19T08:41:48,429 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/recovered.edits/333.seqid to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f/recovered.edits/333.seqid 2024-11-19T08:41:48,430 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,430 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T08:41:48,430 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T08:41:48,431 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T08:41:48,434 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119021fbc256b3f4c07a7d711e6ec90bbcd_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119021fbc256b3f4c07a7d711e6ec90bbcd_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,435 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191aa9e2c20f044214b245c9c537820dd7_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191aa9e2c20f044214b245c9c537820dd7_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,436 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111928c43a06da2c48069f9025a54ed2992a_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111928c43a06da2c48069f9025a54ed2992a_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,437 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111932d6ce1dd5394d02af52bacdd6acf29c_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111932d6ce1dd5394d02af52bacdd6acf29c_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,438 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194132f830b5ce40549a42eaf9f0739822_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194132f830b5ce40549a42eaf9f0739822_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,439 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111949a3016ee3c742ada720117d9ccea150_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111949a3016ee3c742ada720117d9ccea150_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,440 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965b7f9a19ae04a1d8cae9edc52277359_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111965b7f9a19ae04a1d8cae9edc52277359_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,441 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119717ed0250b984d6cba985115c275524b_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119717ed0250b984d6cba985115c275524b_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,442 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111974789ac435584ed1b23624cab665ce2f_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111974789ac435584ed1b23624cab665ce2f_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,443 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111975e4a252c3454ce38a23db6530e4f82a_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111975e4a252c3454ce38a23db6530e4f82a_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,444 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197bbe885858eb4ef2b9f25f6ab09f1cf7_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197bbe885858eb4ef2b9f25f6ab09f1cf7_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,445 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994cdc9ba2f534d3991bdf1424dd04515_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111994cdc9ba2f534d3991bdf1424dd04515_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,447 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afbce7bc068149f5a4a63433b628c983_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afbce7bc068149f5a4a63433b628c983_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,448 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b683500fdb5e408ab25cd130c064cea4_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b683500fdb5e408ab25cd130c064cea4_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,449 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cda646cf60ec4957b17375960c6919a6_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cda646cf60ec4957b17375960c6919a6_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,450 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e346484a8cac48f8bcafffada0abe64c_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e346484a8cac48f8bcafffada0abe64c_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,452 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7d93f54a24f4f6cb6d68d26232ff655_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e7d93f54a24f4f6cb6d68d26232ff655_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,453 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f27239fb3612405484cba86d7407d97c_8f3ff379c750bd3082218a4d8d91a02f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f27239fb3612405484cba86d7407d97c_8f3ff379c750bd3082218a4d8d91a02f 2024-11-19T08:41:48,454 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T08:41:48,456 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:48,459 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T08:41:48,462 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T08:41:48,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:48,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T08:41:48,463 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732005708463"}]},"ts":"9223372036854775807"} 2024-11-19T08:41:48,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T08:41:48,466 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8f3ff379c750bd3082218a4d8d91a02f, NAME => 'TestAcidGuarantees,,1732005680746.8f3ff379c750bd3082218a4d8d91a02f.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T08:41:48,466 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T08:41:48,466 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732005708466"}]},"ts":"9223372036854775807"} 2024-11-19T08:41:48,467 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T08:41:48,468 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:48,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-19T08:41:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T08:41:48,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T08:41:48,506 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:48,520 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=463 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=463 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=5818 (was 6180) 2024-11-19T08:41:48,531 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=5818 2024-11-19T08:41:48,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:41:48,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:41:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T08:41:48,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T08:41:48,535 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:48,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-19T08:41:48,536 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T08:41:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T08:41:48,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742231_1407 (size=959) 2024-11-19T08:41:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T08:41:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T08:41:48,943 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:41:48,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742232_1408 (size=53) 2024-11-19T08:41:49,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T08:41:49,351 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:41:49,352 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e77bbd8dcbf5692aa28a4e1a82d4cbfe, disabling compactions & flushes 2024-11-19T08:41:49,352 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:49,352 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:49,352 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. after waiting 0 ms 2024-11-19T08:41:49,352 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:49,352 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:49,352 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: Waiting for close lock at 1732005709352Disabling compacts and flushes for region at 1732005709352Disabling writes for close at 1732005709352Writing region close event to WAL at 1732005709352Closed at 1732005709352 2024-11-19T08:41:49,355 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T08:41:49,355 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732005709355"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732005709355"}]},"ts":"1732005709355"} 2024-11-19T08:41:49,358 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T08:41:49,359 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T08:41:49,359 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005709359"}]},"ts":"1732005709359"} 2024-11-19T08:41:49,361 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T08:41:49,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, ASSIGN}] 2024-11-19T08:41:49,363 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, ASSIGN 2024-11-19T08:41:49,364 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, ASSIGN; state=OFFLINE, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=false 2024-11-19T08:41:49,514 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=e77bbd8dcbf5692aa28a4e1a82d4cbfe, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:49,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, ASSIGN because future has completed 2024-11-19T08:41:49,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:41:49,674 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:49,674 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:41:49,675 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,675 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:41:49,676 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,676 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T08:41:49,677 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,679 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e77bbd8dcbf5692aa28a4e1a82d4cbfe columnFamilyName A 2024-11-19T08:41:49,679 DEBUG [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:49,680 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] regionserver.HStore(327): Store=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:49,680 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,682 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e77bbd8dcbf5692aa28a4e1a82d4cbfe columnFamilyName B 2024-11-19T08:41:49,682 DEBUG [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:49,682 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] regionserver.HStore(327): Store=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:49,682 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,684 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e77bbd8dcbf5692aa28a4e1a82d4cbfe columnFamilyName C 2024-11-19T08:41:49,684 DEBUG [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:41:49,684 INFO [StoreOpener-e77bbd8dcbf5692aa28a4e1a82d4cbfe-1 {}] regionserver.HStore(327): Store=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:41:49,684 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,685 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,686 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,687 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,687 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,687 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:41:49,689 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,690 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:41:49,691 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened e77bbd8dcbf5692aa28a4e1a82d4cbfe; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63215704, jitterRate=-0.05801260471343994}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:41:49,691 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:49,692 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: Running coprocessor pre-open hook at 1732005709676Writing region info on filesystem at 1732005709676Initializing all the Stores at 1732005709677 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005709677Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005709677Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005709677Cleaning up temporary data from old regions at 1732005709687 (+10 ms)Running coprocessor post-open hooks at 1732005709691 (+4 ms)Region opened successfully at 1732005709692 (+1 ms) 2024-11-19T08:41:49,692 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., pid=126, masterSystemTime=1732005709669 2024-11-19T08:41:49,694 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:49,694 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:49,695 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=e77bbd8dcbf5692aa28a4e1a82d4cbfe, regionState=OPEN, openSeqNum=2, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:49,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:41:49,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T08:41:49,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 in 181 msec 2024-11-19T08:41:49,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-19T08:41:49,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, ASSIGN in 338 msec 2024-11-19T08:41:49,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T08:41:49,703 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005709703"}]},"ts":"1732005709703"} 2024-11-19T08:41:49,704 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T08:41:49,705 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T08:41:49,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-19T08:41:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T08:41:50,686 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:50,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,689 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3988ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330eaf61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,693 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,697 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T08:41:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ab37fa97a98,36325,1732005592414 2024-11-19T08:41:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@669ff470 2024-11-19T08:41:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T08:41:50,700 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T08:41:50,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,701 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ac91dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34842bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,704 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c5e33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,705 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd0d86c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,706 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5370e2c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,707 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8dc2c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,709 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a03ff85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,710 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@507d982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,712 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af84d08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,713 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c549c3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,715 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e23263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,716 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47effa99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,718 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d5112d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,719 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458b2ac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,721 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179972dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,722 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e7a952b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,723 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11505b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,724 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9ad1d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:41:50,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:41:50,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:41:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:41:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:41:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6597ee7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:41:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:41:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:41:50,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:41:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6e53aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:41:50,728 DEBUG [Thread-1846 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,728 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,728 DEBUG [Thread-1855 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,728 DEBUG [Thread-1850 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,729 DEBUG [Thread-1857 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,729 DEBUG [Thread-1844 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,729 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,730 DEBUG [Thread-1844 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,730 DEBUG [Thread-1844 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,730 DEBUG [Thread-1848 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,731 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,731 DEBUG [Thread-1848 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T08:41:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,732 DEBUG [Thread-1852 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:41:50,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T08:41:50,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:41:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:41:50,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,734 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:50,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:41:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:50,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2] 2024-11-19T08:41:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:50,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:50,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:41:50,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/f045afe0d210464998dbaa28cdb3872d is 50, key is test_row_0/A:col10/1732005710741/Put/seqid=0 2024-11-19T08:41:50,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742233_1409 (size=12001) 2024-11-19T08:41:50,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005770770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005770772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/f045afe0d210464998dbaa28cdb3872d 2024-11-19T08:41:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005770773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005770778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005770779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/857391cee2714131863618ea6b677030 is 50, key is test_row_0/B:col10/1732005710741/Put/seqid=0 2024-11-19T08:41:50,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742234_1410 (size=12001) 2024-11-19T08:41:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T08:41:50,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005770886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005770886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005770886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T08:41:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:50,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:50,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:50,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005770896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005770896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T08:41:51,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T08:41:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005771096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005771096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005771096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005771106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005771116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T08:41:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/857391cee2714131863618ea6b677030 2024-11-19T08:41:51,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3f75395c3b5f41faa09142cc2131a9e0 is 50, key is test_row_0/C:col10/1732005710741/Put/seqid=0 2024-11-19T08:41:51,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742235_1411 (size=12001) 2024-11-19T08:41:51,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T08:41:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T08:41:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005771406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005771406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005771406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005771416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005771426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:51,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:51,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T08:41:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:51,525 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T08:41:51,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3f75395c3b5f41faa09142cc2131a9e0 2024-11-19T08:41:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/f045afe0d210464998dbaa28cdb3872d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/f045afe0d210464998dbaa28cdb3872d 2024-11-19T08:41:51,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/f045afe0d210464998dbaa28cdb3872d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T08:41:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/857391cee2714131863618ea6b677030 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/857391cee2714131863618ea6b677030 2024-11-19T08:41:51,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/857391cee2714131863618ea6b677030, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T08:41:51,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3f75395c3b5f41faa09142cc2131a9e0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3f75395c3b5f41faa09142cc2131a9e0 2024-11-19T08:41:51,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3f75395c3b5f41faa09142cc2131a9e0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T08:41:51,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 903ms, sequenceid=14, compaction requested=false 2024-11-19T08:41:51,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:51,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T08:41:51,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,657 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:41:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/58e1a14e20d84b8494ffbb60ba58066e is 50, key is test_row_0/A:col10/1732005710777/Put/seqid=0 2024-11-19T08:41:51,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742236_1412 (size=12001) 2024-11-19T08:41:51,667 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/58e1a14e20d84b8494ffbb60ba58066e 2024-11-19T08:41:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/4a27d30f5c17485ea43895a6efbc86eb is 50, key is test_row_0/B:col10/1732005710777/Put/seqid=0 2024-11-19T08:41:51,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742237_1413 (size=12001) 2024-11-19T08:41:51,679 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/4a27d30f5c17485ea43895a6efbc86eb 2024-11-19T08:41:51,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/4612de0f6ab24468972e59678ed5e730 is 50, key is test_row_0/C:col10/1732005710777/Put/seqid=0 2024-11-19T08:41:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742238_1414 (size=12001) 2024-11-19T08:41:51,692 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/4612de0f6ab24468972e59678ed5e730 2024-11-19T08:41:51,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/58e1a14e20d84b8494ffbb60ba58066e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/58e1a14e20d84b8494ffbb60ba58066e 2024-11-19T08:41:51,700 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/58e1a14e20d84b8494ffbb60ba58066e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T08:41:51,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/4a27d30f5c17485ea43895a6efbc86eb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4a27d30f5c17485ea43895a6efbc86eb 2024-11-19T08:41:51,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4a27d30f5c17485ea43895a6efbc86eb, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T08:41:51,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/4612de0f6ab24468972e59678ed5e730 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4612de0f6ab24468972e59678ed5e730 2024-11-19T08:41:51,713 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4612de0f6ab24468972e59678ed5e730, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T08:41:51,714 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 57ms, sequenceid=37, compaction requested=false 2024-11-19T08:41:51,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:51,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:51,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T08:41:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T08:41:51,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T08:41:51,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 979 msec 2024-11-19T08:41:51,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 989 msec 2024-11-19T08:41:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T08:41:51,866 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:51,867 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T08:41:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T08:41:51,868 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:51,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:51,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:51,953 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:51,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T08:41:51,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/8d09b329011c42f9aa8ab015cede5d0b is 50, key is test_row_0/A:col10/1732005711922/Put/seqid=0 2024-11-19T08:41:51,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742239_1415 (size=12001) 2024-11-19T08:41:51,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/8d09b329011c42f9aa8ab015cede5d0b 2024-11-19T08:41:51,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/ba8d72f49e2c433fb426aa2383e14adb is 50, key is test_row_0/B:col10/1732005711922/Put/seqid=0 2024-11-19T08:41:51,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742240_1416 (size=12001) 2024-11-19T08:41:51,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/ba8d72f49e2c433fb426aa2383e14adb 2024-11-19T08:41:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T08:41:51,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/07ebf2b6b4f643beadbfc9bb45520ad3 is 50, key is test_row_0/C:col10/1732005711922/Put/seqid=0 2024-11-19T08:41:51,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742241_1417 (size=12001) 2024-11-19T08:41:52,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005772001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005772008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005772009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005772011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005772012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T08:41:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:52,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005772116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005772116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005772126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005772126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005772126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T08:41:52,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:52,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T08:41:52,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T08:41:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:52,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005772326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005772326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005772336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005772336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005772336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/07ebf2b6b4f643beadbfc9bb45520ad3 2024-11-19T08:41:52,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/8d09b329011c42f9aa8ab015cede5d0b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8d09b329011c42f9aa8ab015cede5d0b 2024-11-19T08:41:52,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8d09b329011c42f9aa8ab015cede5d0b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T08:41:52,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/ba8d72f49e2c433fb426aa2383e14adb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba8d72f49e2c433fb426aa2383e14adb 2024-11-19T08:41:52,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba8d72f49e2c433fb426aa2383e14adb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T08:41:52,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/07ebf2b6b4f643beadbfc9bb45520ad3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/07ebf2b6b4f643beadbfc9bb45520ad3 2024-11-19T08:41:52,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/07ebf2b6b4f643beadbfc9bb45520ad3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T08:41:52,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 442ms, sequenceid=50, compaction requested=true 2024-11-19T08:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:52,396 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:52,396 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:52,397 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:52,397 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:52,397 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:41:52,397 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:41:52,397 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,397 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,397 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/f045afe0d210464998dbaa28cdb3872d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/58e1a14e20d84b8494ffbb60ba58066e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8d09b329011c42f9aa8ab015cede5d0b] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.2 K 2024-11-19T08:41:52,397 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/857391cee2714131863618ea6b677030, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4a27d30f5c17485ea43895a6efbc86eb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba8d72f49e2c433fb426aa2383e14adb] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.2 K 2024-11-19T08:41:52,397 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting f045afe0d210464998dbaa28cdb3872d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732005710741 2024-11-19T08:41:52,397 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 857391cee2714131863618ea6b677030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732005710741 2024-11-19T08:41:52,397 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58e1a14e20d84b8494ffbb60ba58066e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732005710766 2024-11-19T08:41:52,398 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a27d30f5c17485ea43895a6efbc86eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732005710766 2024-11-19T08:41:52,398 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d09b329011c42f9aa8ab015cede5d0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732005711922 2024-11-19T08:41:52,398 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ba8d72f49e2c433fb426aa2383e14adb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732005711922 2024-11-19T08:41:52,403 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:52,403 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6c06805dba5048c89ce0e84f18771a90 is 50, key is test_row_0/B:col10/1732005711922/Put/seqid=0 2024-11-19T08:41:52,405 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:52,406 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/8487efa376914322869f0664b7f2ed65 is 50, key is test_row_0/A:col10/1732005711922/Put/seqid=0 2024-11-19T08:41:52,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742242_1418 (size=12104) 2024-11-19T08:41:52,421 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6c06805dba5048c89ce0e84f18771a90 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6c06805dba5048c89ce0e84f18771a90 2024-11-19T08:41:52,427 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 6c06805dba5048c89ce0e84f18771a90(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:52,427 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:52,427 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=13, startTime=1732005712396; duration=0sec 2024-11-19T08:41:52,427 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:52,427 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:41:52,427 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:52,428 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:52,428 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:41:52,429 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,429 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3f75395c3b5f41faa09142cc2131a9e0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4612de0f6ab24468972e59678ed5e730, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/07ebf2b6b4f643beadbfc9bb45520ad3] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.2 K 2024-11-19T08:41:52,429 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f75395c3b5f41faa09142cc2131a9e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732005710741 2024-11-19T08:41:52,429 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4612de0f6ab24468972e59678ed5e730, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732005710766 2024-11-19T08:41:52,430 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ebf2b6b4f643beadbfc9bb45520ad3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732005711922 2024-11-19T08:41:52,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742243_1419 (size=12104) 2024-11-19T08:41:52,436 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:52,436 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/4c2ab3cc83074b4295d632752e53515c is 50, key is test_row_0/C:col10/1732005711922/Put/seqid=0 2024-11-19T08:41:52,444 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/8487efa376914322869f0664b7f2ed65 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8487efa376914322869f0664b7f2ed65 2024-11-19T08:41:52,449 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 8487efa376914322869f0664b7f2ed65(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:52,449 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:52,449 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=13, startTime=1732005712396; duration=0sec 2024-11-19T08:41:52,450 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:52,450 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:41:52,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742244_1420 (size=12104) 2024-11-19T08:41:52,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T08:41:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,478 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:41:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/274b78354f774c1887b49a53cd270154 is 50, key is test_row_0/A:col10/1732005711999/Put/seqid=0 2024-11-19T08:41:52,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742245_1421 (size=12001) 2024-11-19T08:41:52,494 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/274b78354f774c1887b49a53cd270154 2024-11-19T08:41:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T08:41:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/b709d03114104b3c8cec2ac6e285aee8 is 50, key is test_row_0/B:col10/1732005711999/Put/seqid=0 2024-11-19T08:41:52,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742246_1422 (size=12001) 2024-11-19T08:41:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:52,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005772656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005772657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005772660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005772664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005772665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005772776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005772776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005772776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005772786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:52,859 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/4c2ab3cc83074b4295d632752e53515c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4c2ab3cc83074b4295d632752e53515c 2024-11-19T08:41:52,863 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 4c2ab3cc83074b4295d632752e53515c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:52,863 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:52,863 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=13, startTime=1732005712396; duration=0sec 2024-11-19T08:41:52,863 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:52,863 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:41:52,912 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/b709d03114104b3c8cec2ac6e285aee8 2024-11-19T08:41:52,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/b5d0f7ea49d045739d6af185cea332a0 is 50, key is test_row_0/C:col10/1732005711999/Put/seqid=0 2024-11-19T08:41:52,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742247_1423 (size=12001) 2024-11-19T08:41:52,922 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/b5d0f7ea49d045739d6af185cea332a0 2024-11-19T08:41:52,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/274b78354f774c1887b49a53cd270154 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/274b78354f774c1887b49a53cd270154 2024-11-19T08:41:52,929 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/274b78354f774c1887b49a53cd270154, entries=150, sequenceid=75, filesize=11.7 K 2024-11-19T08:41:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/b709d03114104b3c8cec2ac6e285aee8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b709d03114104b3c8cec2ac6e285aee8 2024-11-19T08:41:52,932 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b709d03114104b3c8cec2ac6e285aee8, entries=150, sequenceid=75, filesize=11.7 K 2024-11-19T08:41:52,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/b5d0f7ea49d045739d6af185cea332a0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b5d0f7ea49d045739d6af185cea332a0 2024-11-19T08:41:52,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b5d0f7ea49d045739d6af185cea332a0, entries=150, sequenceid=75, filesize=11.7 K 2024-11-19T08:41:52,937 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 458ms, sequenceid=75, compaction requested=false 2024-11-19T08:41:52,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:52,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:52,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T08:41:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T08:41:52,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T08:41:52,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-19T08:41:52,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0730 sec 2024-11-19T08:41:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:52,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:52,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T08:41:52,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/26c88a4ea8034e0a9472e672b0452e16 is 50, key is test_row_0/A:col10/1732005712991/Put/seqid=0 2024-11-19T08:41:52,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742248_1424 (size=16681) 2024-11-19T08:41:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T08:41:53,006 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:53,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T08:41:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T08:41:53,007 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:53,008 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:53,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:53,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005773025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005773032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005773034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005773035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T08:41:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005773146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005773146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005773146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005773155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:53,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:53,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005773176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:53,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:53,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T08:41:53,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005773356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005773356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005773356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005773366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/26c88a4ea8034e0a9472e672b0452e16 2024-11-19T08:41:53,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/ba739cfff8c647ce9da7f88f78254688 is 50, key is test_row_0/B:col10/1732005712991/Put/seqid=0 2024-11-19T08:41:53,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742249_1425 (size=12001) 2024-11-19T08:41:53,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:53,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T08:41:53,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005773675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005773675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005773676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005773686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:53,741 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T08:41:53,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/ba739cfff8c647ce9da7f88f78254688 2024-11-19T08:41:53,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3b2d614910014daf9346aa2c7b6a03ea is 50, key is test_row_0/C:col10/1732005712991/Put/seqid=0 2024-11-19T08:41:53,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742250_1426 (size=12001) 2024-11-19T08:41:53,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:53,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:54,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:54,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T08:41:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005774185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005774186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005774186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:54,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005774196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:54,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005774196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:54,204 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:54,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3b2d614910014daf9346aa2c7b6a03ea 2024-11-19T08:41:54,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/26c88a4ea8034e0a9472e672b0452e16 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/26c88a4ea8034e0a9472e672b0452e16 2024-11-19T08:41:54,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/26c88a4ea8034e0a9472e672b0452e16, entries=250, sequenceid=91, filesize=16.3 K 2024-11-19T08:41:54,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/ba739cfff8c647ce9da7f88f78254688 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba739cfff8c647ce9da7f88f78254688 2024-11-19T08:41:54,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba739cfff8c647ce9da7f88f78254688, entries=150, sequenceid=91, filesize=11.7 K 2024-11-19T08:41:54,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3b2d614910014daf9346aa2c7b6a03ea as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3b2d614910014daf9346aa2c7b6a03ea 2024-11-19T08:41:54,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:54,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3b2d614910014daf9346aa2c7b6a03ea, entries=150, sequenceid=91, filesize=11.7 K 2024-11-19T08:41:54,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1242ms, sequenceid=91, compaction requested=true 2024-11-19T08:41:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:54,233 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:54,233 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:54,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:41:54,234 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,234 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,234 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6c06805dba5048c89ce0e84f18771a90, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b709d03114104b3c8cec2ac6e285aee8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba739cfff8c647ce9da7f88f78254688] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.3 K 2024-11-19T08:41:54,234 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8487efa376914322869f0664b7f2ed65, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/274b78354f774c1887b49a53cd270154, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/26c88a4ea8034e0a9472e672b0452e16] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=39.8 K 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c06805dba5048c89ce0e84f18771a90, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732005711922 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8487efa376914322869f0664b7f2ed65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732005711922 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b709d03114104b3c8cec2ac6e285aee8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732005711999 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 274b78354f774c1887b49a53cd270154, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732005711999 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ba739cfff8c647ce9da7f88f78254688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732005712663 2024-11-19T08:41:54,234 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26c88a4ea8034e0a9472e672b0452e16, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732005712659 2024-11-19T08:41:54,241 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:54,242 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:54,242 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/0a2e380f329940e8bd23e0e93806981f is 50, key is test_row_0/B:col10/1732005712991/Put/seqid=0 2024-11-19T08:41:54,242 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/67b1a5261da249afaeeaec4e34bede94 is 50, key is test_row_0/A:col10/1732005712991/Put/seqid=0 2024-11-19T08:41:54,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742251_1427 (size=12207) 2024-11-19T08:41:54,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742252_1428 (size=12207) 2024-11-19T08:41:54,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T08:41:54,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,383 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T08:41:54,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/7134dfbd38164d8eb04d769b391745cb is 50, key is test_row_0/A:col10/1732005713033/Put/seqid=0 2024-11-19T08:41:54,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742253_1429 (size=12001) 2024-11-19T08:41:54,657 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/67b1a5261da249afaeeaec4e34bede94 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/67b1a5261da249afaeeaec4e34bede94 2024-11-19T08:41:54,658 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/0a2e380f329940e8bd23e0e93806981f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/0a2e380f329940e8bd23e0e93806981f 2024-11-19T08:41:54,663 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 67b1a5261da249afaeeaec4e34bede94(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:54,663 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 0a2e380f329940e8bd23e0e93806981f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:54,663 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:54,663 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:54,663 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=13, startTime=1732005714233; duration=0sec 2024-11-19T08:41:54,663 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=13, startTime=1732005714233; duration=0sec 2024-11-19T08:41:54,663 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:54,663 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:41:54,663 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:54,663 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:41:54,663 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:54,664 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:54,664 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:41:54,664 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:54,664 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4c2ab3cc83074b4295d632752e53515c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b5d0f7ea49d045739d6af185cea332a0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3b2d614910014daf9346aa2c7b6a03ea] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.3 K 2024-11-19T08:41:54,664 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c2ab3cc83074b4295d632752e53515c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732005711922 2024-11-19T08:41:54,664 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d0f7ea49d045739d6af185cea332a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732005711999 2024-11-19T08:41:54,665 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2d614910014daf9346aa2c7b6a03ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732005712663 2024-11-19T08:41:54,671 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:54,671 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/a162c8cb560d45a2943f98c8e35d035b is 50, key is test_row_0/C:col10/1732005712991/Put/seqid=0 2024-11-19T08:41:54,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742254_1430 (size=12207) 2024-11-19T08:41:54,687 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/a162c8cb560d45a2943f98c8e35d035b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/a162c8cb560d45a2943f98c8e35d035b 2024-11-19T08:41:54,693 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into a162c8cb560d45a2943f98c8e35d035b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:54,693 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:54,693 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=13, startTime=1732005714233; duration=0sec 2024-11-19T08:41:54,693 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:54,693 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:41:54,791 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/7134dfbd38164d8eb04d769b391745cb 2024-11-19T08:41:54,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/166aaba028f14756bae28273ddda7614 is 50, key is test_row_0/B:col10/1732005713033/Put/seqid=0 2024-11-19T08:41:54,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742255_1431 (size=12001) 2024-11-19T08:41:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T08:41:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005775220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005775226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005775227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005775228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,248 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/166aaba028f14756bae28273ddda7614 2024-11-19T08:41:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/5171c5ed02fd4201872744a6257e0071 is 50, key is test_row_0/C:col10/1732005713033/Put/seqid=0 2024-11-19T08:41:55,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742256_1432 (size=12001) 2024-11-19T08:41:55,261 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/5171c5ed02fd4201872744a6257e0071 2024-11-19T08:41:55,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/7134dfbd38164d8eb04d769b391745cb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7134dfbd38164d8eb04d769b391745cb 2024-11-19T08:41:55,271 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7134dfbd38164d8eb04d769b391745cb, entries=150, sequenceid=112, filesize=11.7 K 2024-11-19T08:41:55,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/166aaba028f14756bae28273ddda7614 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/166aaba028f14756bae28273ddda7614 2024-11-19T08:41:55,275 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/166aaba028f14756bae28273ddda7614, entries=150, sequenceid=112, filesize=11.7 K 2024-11-19T08:41:55,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/5171c5ed02fd4201872744a6257e0071 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5171c5ed02fd4201872744a6257e0071 2024-11-19T08:41:55,282 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5171c5ed02fd4201872744a6257e0071, entries=150, sequenceid=112, filesize=11.7 K 2024-11-19T08:41:55,282 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 899ms, sequenceid=112, compaction requested=false 2024-11-19T08:41:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:55,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T08:41:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T08:41:55,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T08:41:55,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2780 sec 2024-11-19T08:41:55,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.2820 sec 2024-11-19T08:41:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:55,341 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:55,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T08:41:55,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/ab72c14e8c284447ad6b9e0b2807cf49 is 50, key is test_row_0/A:col10/1732005715340/Put/seqid=0 2024-11-19T08:41:55,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742257_1433 (size=14441) 2024-11-19T08:41:55,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005775376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005775378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005775386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005775387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005775496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005775496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005775496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005775506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005775706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005775716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005775716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005775716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:55,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/ab72c14e8c284447ad6b9e0b2807cf49 2024-11-19T08:41:55,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6b5e36efa5184986b4ffa91d0dbf0923 is 50, key is test_row_0/B:col10/1732005715340/Put/seqid=0 2024-11-19T08:41:55,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742258_1434 (size=12051) 2024-11-19T08:41:56,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005776015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005776026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005776026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005776036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6b5e36efa5184986b4ffa91d0dbf0923 2024-11-19T08:41:56,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/e58ab71d114b46a7bcb7b34a0033834f is 50, key is test_row_0/C:col10/1732005715340/Put/seqid=0 2024-11-19T08:41:56,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742259_1435 (size=12051) 2024-11-19T08:41:56,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005776216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,223 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,384 WARN [regionserver/3ab37fa97a98:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-19T08:41:56,385 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/f045afe0d210464998dbaa28cdb3872d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/58e1a14e20d84b8494ffbb60ba58066e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8487efa376914322869f0664b7f2ed65, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8d09b329011c42f9aa8ab015cede5d0b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/274b78354f774c1887b49a53cd270154, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/26c88a4ea8034e0a9472e672b0452e16] to archive 2024-11-19T08:41:56,386 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:56,387 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/f045afe0d210464998dbaa28cdb3872d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/f045afe0d210464998dbaa28cdb3872d 2024-11-19T08:41:56,388 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/58e1a14e20d84b8494ffbb60ba58066e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/58e1a14e20d84b8494ffbb60ba58066e 2024-11-19T08:41:56,389 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8487efa376914322869f0664b7f2ed65 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8487efa376914322869f0664b7f2ed65 2024-11-19T08:41:56,390 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8d09b329011c42f9aa8ab015cede5d0b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8d09b329011c42f9aa8ab015cede5d0b 2024-11-19T08:41:56,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/274b78354f774c1887b49a53cd270154 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/274b78354f774c1887b49a53cd270154 2024-11-19T08:41:56,391 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/26c88a4ea8034e0a9472e672b0452e16 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/26c88a4ea8034e0a9472e672b0452e16 2024-11-19T08:41:56,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/857391cee2714131863618ea6b677030, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4a27d30f5c17485ea43895a6efbc86eb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6c06805dba5048c89ce0e84f18771a90, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba8d72f49e2c433fb426aa2383e14adb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b709d03114104b3c8cec2ac6e285aee8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba739cfff8c647ce9da7f88f78254688] to archive 2024-11-19T08:41:56,393 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:56,394 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/857391cee2714131863618ea6b677030 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/857391cee2714131863618ea6b677030 2024-11-19T08:41:56,395 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4a27d30f5c17485ea43895a6efbc86eb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4a27d30f5c17485ea43895a6efbc86eb 2024-11-19T08:41:56,396 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6c06805dba5048c89ce0e84f18771a90 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6c06805dba5048c89ce0e84f18771a90 2024-11-19T08:41:56,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba8d72f49e2c433fb426aa2383e14adb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba8d72f49e2c433fb426aa2383e14adb 2024-11-19T08:41:56,397 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b709d03114104b3c8cec2ac6e285aee8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b709d03114104b3c8cec2ac6e285aee8 2024-11-19T08:41:56,398 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba739cfff8c647ce9da7f88f78254688 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/ba739cfff8c647ce9da7f88f78254688 2024-11-19T08:41:56,399 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3f75395c3b5f41faa09142cc2131a9e0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4612de0f6ab24468972e59678ed5e730, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4c2ab3cc83074b4295d632752e53515c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/07ebf2b6b4f643beadbfc9bb45520ad3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b5d0f7ea49d045739d6af185cea332a0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3b2d614910014daf9346aa2c7b6a03ea] to archive 2024-11-19T08:41:56,400 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:41:56,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3f75395c3b5f41faa09142cc2131a9e0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3f75395c3b5f41faa09142cc2131a9e0 2024-11-19T08:41:56,401 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4612de0f6ab24468972e59678ed5e730 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4612de0f6ab24468972e59678ed5e730 2024-11-19T08:41:56,402 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4c2ab3cc83074b4295d632752e53515c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/4c2ab3cc83074b4295d632752e53515c 2024-11-19T08:41:56,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/07ebf2b6b4f643beadbfc9bb45520ad3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/07ebf2b6b4f643beadbfc9bb45520ad3 2024-11-19T08:41:56,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b5d0f7ea49d045739d6af185cea332a0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b5d0f7ea49d045739d6af185cea332a0 2024-11-19T08:41:56,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3ab37fa97a98:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3b2d614910014daf9346aa2c7b6a03ea to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3b2d614910014daf9346aa2c7b6a03ea 2024-11-19T08:41:56,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005776526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005776536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005776536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005776556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:56,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/e58ab71d114b46a7bcb7b34a0033834f 2024-11-19T08:41:56,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/ab72c14e8c284447ad6b9e0b2807cf49 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/ab72c14e8c284447ad6b9e0b2807cf49 2024-11-19T08:41:56,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/ab72c14e8c284447ad6b9e0b2807cf49, entries=200, sequenceid=131, filesize=14.1 K 2024-11-19T08:41:56,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6b5e36efa5184986b4ffa91d0dbf0923 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6b5e36efa5184986b4ffa91d0dbf0923 2024-11-19T08:41:56,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6b5e36efa5184986b4ffa91d0dbf0923, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T08:41:56,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/e58ab71d114b46a7bcb7b34a0033834f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e58ab71d114b46a7bcb7b34a0033834f 2024-11-19T08:41:56,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e58ab71d114b46a7bcb7b34a0033834f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T08:41:56,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1241ms, sequenceid=131, compaction requested=true 2024-11-19T08:41:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:56,582 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:56,582 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:56,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:56,583 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:56,583 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:56,583 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:41:56,583 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:41:56,583 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:56,583 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:56,583 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/0a2e380f329940e8bd23e0e93806981f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/166aaba028f14756bae28273ddda7614, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6b5e36efa5184986b4ffa91d0dbf0923] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.4 K 2024-11-19T08:41:56,583 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/67b1a5261da249afaeeaec4e34bede94, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7134dfbd38164d8eb04d769b391745cb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/ab72c14e8c284447ad6b9e0b2807cf49] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=37.7 K 2024-11-19T08:41:56,583 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a2e380f329940e8bd23e0e93806981f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732005712663 2024-11-19T08:41:56,583 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67b1a5261da249afaeeaec4e34bede94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732005712663 2024-11-19T08:41:56,584 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7134dfbd38164d8eb04d769b391745cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732005713022 2024-11-19T08:41:56,584 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 166aaba028f14756bae28273ddda7614, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732005713022 2024-11-19T08:41:56,584 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab72c14e8c284447ad6b9e0b2807cf49, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732005715213 2024-11-19T08:41:56,584 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b5e36efa5184986b4ffa91d0dbf0923, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732005715213 2024-11-19T08:41:56,592 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:56,593 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1deb73da926e4ca4bf382dd7cd80c808 is 50, key is test_row_0/B:col10/1732005715340/Put/seqid=0 2024-11-19T08:41:56,593 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:56,593 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/5f32f68d4d7c4d109005a3e09c053675 is 50, key is test_row_0/A:col10/1732005715340/Put/seqid=0 2024-11-19T08:41:56,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742260_1436 (size=12154) 2024-11-19T08:41:56,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742261_1437 (size=12154) 2024-11-19T08:41:56,607 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1deb73da926e4ca4bf382dd7cd80c808 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1deb73da926e4ca4bf382dd7cd80c808 2024-11-19T08:41:56,610 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 1deb73da926e4ca4bf382dd7cd80c808(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:56,611 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:56,611 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=13, startTime=1732005716582; duration=0sec 2024-11-19T08:41:56,611 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:56,611 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:41:56,611 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:56,612 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:56,612 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:41:56,612 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:56,612 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/a162c8cb560d45a2943f98c8e35d035b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5171c5ed02fd4201872744a6257e0071, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e58ab71d114b46a7bcb7b34a0033834f] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.4 K 2024-11-19T08:41:56,612 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a162c8cb560d45a2943f98c8e35d035b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732005712663 2024-11-19T08:41:56,613 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5171c5ed02fd4201872744a6257e0071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732005713022 2024-11-19T08:41:56,613 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting e58ab71d114b46a7bcb7b34a0033834f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732005715213 2024-11-19T08:41:56,618 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#368 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:56,618 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/53dcfb18d5ed497fb291d6163a42c86d is 50, key is test_row_0/C:col10/1732005715340/Put/seqid=0 2024-11-19T08:41:56,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742262_1438 (size=12154) 2024-11-19T08:41:57,008 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/5f32f68d4d7c4d109005a3e09c053675 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5f32f68d4d7c4d109005a3e09c053675 2024-11-19T08:41:57,011 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 5f32f68d4d7c4d109005a3e09c053675(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:57,011 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:57,011 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=13, startTime=1732005716582; duration=0sec 2024-11-19T08:41:57,011 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:57,011 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:41:57,028 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/53dcfb18d5ed497fb291d6163a42c86d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/53dcfb18d5ed497fb291d6163a42c86d 2024-11-19T08:41:57,041 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 53dcfb18d5ed497fb291d6163a42c86d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:41:57,041 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:57,041 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=13, startTime=1732005716582; duration=0sec 2024-11-19T08:41:57,041 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:57,041 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:41:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T08:41:57,167 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:57,167 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-19T08:41:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T08:41:57,168 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:57,169 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:57,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T08:41:57,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T08:41:57,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:57,321 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T08:41:57,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/3458e8d5f24b4722bb296152b3fb5eee is 50, key is test_row_0/A:col10/1732005715384/Put/seqid=0 2024-11-19T08:41:57,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742263_1439 (size=12151) 2024-11-19T08:41:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T08:41:57,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005777576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005777576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005777581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,587 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005777582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005777696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005777696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005777696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,729 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/3458e8d5f24b4722bb296152b3fb5eee 2024-11-19T08:41:57,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/3e500db4a26b49879e24a99cbd49118a is 50, key is test_row_0/B:col10/1732005715384/Put/seqid=0 2024-11-19T08:41:57,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742264_1440 (size=12151) 2024-11-19T08:41:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T08:41:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005777906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005777906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:57,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005777916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/3e500db4a26b49879e24a99cbd49118a 2024-11-19T08:41:58,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3d449c1fedc94cdda7773050f7290735 is 50, key is test_row_0/C:col10/1732005715384/Put/seqid=0 2024-11-19T08:41:58,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742265_1441 (size=12151) 2024-11-19T08:41:58,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005778216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005778216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005778236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T08:41:58,547 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3d449c1fedc94cdda7773050f7290735 2024-11-19T08:41:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/3458e8d5f24b4722bb296152b3fb5eee as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/3458e8d5f24b4722bb296152b3fb5eee 2024-11-19T08:41:58,553 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/3458e8d5f24b4722bb296152b3fb5eee, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T08:41:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/3e500db4a26b49879e24a99cbd49118a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3e500db4a26b49879e24a99cbd49118a 2024-11-19T08:41:58,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3e500db4a26b49879e24a99cbd49118a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T08:41:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/3d449c1fedc94cdda7773050f7290735 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3d449c1fedc94cdda7773050f7290735 2024-11-19T08:41:58,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3d449c1fedc94cdda7773050f7290735, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T08:41:58,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1239ms, sequenceid=154, compaction requested=false 2024-11-19T08:41:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-19T08:41:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-19T08:41:58,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T08:41:58,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3930 sec 2024-11-19T08:41:58,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3970 sec 2024-11-19T08:41:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:41:58,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:41:58,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T08:41:58,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/5a369fd541f64d938d43b16e92e8e088 is 50, key is test_row_0/A:col10/1732005717580/Put/seqid=0 2024-11-19T08:41:58,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742266_1442 (size=16931) 2024-11-19T08:41:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005778770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005778777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005778778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005778886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005778886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:58,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005778896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005779096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005779096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005779116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/5a369fd541f64d938d43b16e92e8e088 2024-11-19T08:41:59,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/3c76fa1e3e8b41aba23dfeb3d808b07e is 50, key is test_row_0/B:col10/1732005717580/Put/seqid=0 2024-11-19T08:41:59,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742267_1443 (size=12151) 2024-11-19T08:41:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T08:41:59,316 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:41:59,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:41:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-19T08:41:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T08:41:59,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:41:59,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:41:59,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:41:59,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005779405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005779416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T08:41:59,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005779426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T08:41:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/3c76fa1e3e8b41aba23dfeb3d808b07e 2024-11-19T08:41:59,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/06752a737cdc41cdace50dfb24e69b28 is 50, key is test_row_0/C:col10/1732005717580/Put/seqid=0 2024-11-19T08:41:59,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742268_1444 (size=12151) 2024-11-19T08:41:59,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005779605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,611 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:41:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T08:41:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T08:41:59,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T08:41:59,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005779916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T08:41:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:41:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:41:59,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005779936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T08:41:59,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:41:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005779945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:41:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:41:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:41:59,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/06752a737cdc41cdace50dfb24e69b28 2024-11-19T08:41:59,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/5a369fd541f64d938d43b16e92e8e088 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5a369fd541f64d938d43b16e92e8e088 2024-11-19T08:41:59,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5a369fd541f64d938d43b16e92e8e088, entries=250, sequenceid=172, filesize=16.5 K 2024-11-19T08:41:59,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/3c76fa1e3e8b41aba23dfeb3d808b07e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3c76fa1e3e8b41aba23dfeb3d808b07e 2024-11-19T08:41:59,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3c76fa1e3e8b41aba23dfeb3d808b07e, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T08:41:59,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/06752a737cdc41cdace50dfb24e69b28 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/06752a737cdc41cdace50dfb24e69b28 2024-11-19T08:41:59,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/06752a737cdc41cdace50dfb24e69b28, entries=150, sequenceid=172, filesize=11.9 K 2024-11-19T08:41:59,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1241ms, sequenceid=172, compaction requested=true 2024-11-19T08:41:59,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:41:59,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:41:59,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:59,970 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:59,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:41:59,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:41:59,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:41:59,970 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:41:59,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:41:59,971 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36456 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:59,971 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41236 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:41:59,971 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:41:59,971 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:41:59,971 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,971 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:41:59,971 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5f32f68d4d7c4d109005a3e09c053675, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/3458e8d5f24b4722bb296152b3fb5eee, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5a369fd541f64d938d43b16e92e8e088] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=40.3 K 2024-11-19T08:41:59,971 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1deb73da926e4ca4bf382dd7cd80c808, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3e500db4a26b49879e24a99cbd49118a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3c76fa1e3e8b41aba23dfeb3d808b07e] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.6 K 2024-11-19T08:41:59,972 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f32f68d4d7c4d109005a3e09c053675, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732005715213 2024-11-19T08:41:59,972 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 1deb73da926e4ca4bf382dd7cd80c808, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732005715213 2024-11-19T08:41:59,972 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3458e8d5f24b4722bb296152b3fb5eee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732005715373 2024-11-19T08:41:59,972 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e500db4a26b49879e24a99cbd49118a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732005715373 2024-11-19T08:41:59,972 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a369fd541f64d938d43b16e92e8e088, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732005717573 2024-11-19T08:41:59,972 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c76fa1e3e8b41aba23dfeb3d808b07e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732005717573 2024-11-19T08:41:59,977 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:59,977 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:41:59,978 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/8f320f06ae7c401bba9462a7c744ce97 is 50, key is test_row_0/A:col10/1732005717580/Put/seqid=0 2024-11-19T08:41:59,978 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1d29f092708f4ca5af9342340f21c08c is 50, key is test_row_0/B:col10/1732005717580/Put/seqid=0 2024-11-19T08:41:59,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742269_1445 (size=12357) 2024-11-19T08:41:59,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742270_1446 (size=12357) 2024-11-19T08:42:00,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T08:42:00,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:00,081 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T08:42:00,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/7a714430fafd488c9df0e078d75f03a0 is 50, key is test_row_0/A:col10/1732005718777/Put/seqid=0 2024-11-19T08:42:00,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742271_1447 (size=12151) 2024-11-19T08:42:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:00,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:00,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005780351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:00,385 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1d29f092708f4ca5af9342340f21c08c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1d29f092708f4ca5af9342340f21c08c 2024-11-19T08:42:00,389 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 1d29f092708f4ca5af9342340f21c08c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:00,389 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:00,389 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=13, startTime=1732005719970; duration=0sec 2024-11-19T08:42:00,389 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:00,389 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:42:00,389 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:00,389 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36456 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:00,390 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:42:00,390 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:00,390 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/53dcfb18d5ed497fb291d6163a42c86d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3d449c1fedc94cdda7773050f7290735, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/06752a737cdc41cdace50dfb24e69b28] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.6 K 2024-11-19T08:42:00,390 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 53dcfb18d5ed497fb291d6163a42c86d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732005715213 2024-11-19T08:42:00,390 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d449c1fedc94cdda7773050f7290735, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732005715373 2024-11-19T08:42:00,390 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 06752a737cdc41cdace50dfb24e69b28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732005717573 2024-11-19T08:42:00,398 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:00,399 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/e3731fe6b3a940a884751491c74af758 is 50, key is test_row_0/C:col10/1732005717580/Put/seqid=0 2024-11-19T08:42:00,401 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/8f320f06ae7c401bba9462a7c744ce97 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8f320f06ae7c401bba9462a7c744ce97 2024-11-19T08:42:00,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742272_1448 (size=12357) 2024-11-19T08:42:00,407 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 8f320f06ae7c401bba9462a7c744ce97(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:00,407 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:00,408 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=13, startTime=1732005719970; duration=0sec 2024-11-19T08:42:00,408 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:00,408 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:42:00,408 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/e3731fe6b3a940a884751491c74af758 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e3731fe6b3a940a884751491c74af758 2024-11-19T08:42:00,411 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into e3731fe6b3a940a884751491c74af758(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:00,411 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:00,411 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=13, startTime=1732005719970; duration=0sec 2024-11-19T08:42:00,411 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:00,411 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:42:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T08:42:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005780465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:00,491 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/7a714430fafd488c9df0e078d75f03a0 2024-11-19T08:42:00,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/129578408ca747cbb1a05d95922ab8c0 is 50, key is test_row_0/B:col10/1732005718777/Put/seqid=0 2024-11-19T08:42:00,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742273_1449 (size=12151) 2024-11-19T08:42:00,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005780676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:00,901 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/129578408ca747cbb1a05d95922ab8c0 2024-11-19T08:42:00,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/5518731d355a487191bb6ca3e2204d62 is 50, key is test_row_0/C:col10/1732005718777/Put/seqid=0 2024-11-19T08:42:00,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742274_1450 (size=12151) 2024-11-19T08:42:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005780936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:00,938 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:00,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005780945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:00,947 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:00,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005780966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:00,971 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:00,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005780986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:00,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:01,309 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/5518731d355a487191bb6ca3e2204d62 2024-11-19T08:42:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/7a714430fafd488c9df0e078d75f03a0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7a714430fafd488c9df0e078d75f03a0 2024-11-19T08:42:01,316 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7a714430fafd488c9df0e078d75f03a0, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T08:42:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/129578408ca747cbb1a05d95922ab8c0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/129578408ca747cbb1a05d95922ab8c0 2024-11-19T08:42:01,320 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/129578408ca747cbb1a05d95922ab8c0, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T08:42:01,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/5518731d355a487191bb6ca3e2204d62 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5518731d355a487191bb6ca3e2204d62 2024-11-19T08:42:01,323 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5518731d355a487191bb6ca3e2204d62, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T08:42:01,324 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1243ms, sequenceid=190, compaction requested=false 2024-11-19T08:42:01,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:01,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-19T08:42:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-19T08:42:01,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T08:42:01,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0070 sec 2024-11-19T08:42:01,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.0120 sec 2024-11-19T08:42:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T08:42:01,466 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:01,466 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-19T08:42:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T08:42:01,468 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:01,468 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:01,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:01,510 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:01,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T08:42:01,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d is 50, key is test_row_0/A:col10/1732005720350/Put/seqid=0 2024-11-19T08:42:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742275_1451 (size=14541) 2024-11-19T08:42:01,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d 2024-11-19T08:42:01,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/7e12487971294800990af68c58a0cf91 is 50, key is test_row_0/B:col10/1732005720350/Put/seqid=0 2024-11-19T08:42:01,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742276_1452 (size=12151) 2024-11-19T08:42:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T08:42:01,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005781608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:01,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T08:42:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:01,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005781716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:01,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T08:42:01,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:01,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T08:42:01,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T08:42:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:01,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/7e12487971294800990af68c58a0cf91 2024-11-19T08:42:01,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/c4ea581fbcea4162a83177476053bacd is 50, key is test_row_0/C:col10/1732005720350/Put/seqid=0 2024-11-19T08:42:01,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005781925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:01,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742277_1453 (size=12151) 2024-11-19T08:42:01,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/c4ea581fbcea4162a83177476053bacd 2024-11-19T08:42:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d 2024-11-19T08:42:01,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d, entries=200, sequenceid=212, filesize=14.2 K 2024-11-19T08:42:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/7e12487971294800990af68c58a0cf91 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/7e12487971294800990af68c58a0cf91 2024-11-19T08:42:01,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/7e12487971294800990af68c58a0cf91, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T08:42:01,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/c4ea581fbcea4162a83177476053bacd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c4ea581fbcea4162a83177476053bacd 2024-11-19T08:42:01,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c4ea581fbcea4162a83177476053bacd, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T08:42:01,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 442ms, sequenceid=212, compaction requested=true 2024-11-19T08:42:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:01,952 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:01,952 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:01,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:01,953 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:01,953 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:01,954 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:42:01,954 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:42:01,954 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,954 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:01,954 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1d29f092708f4ca5af9342340f21c08c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/129578408ca747cbb1a05d95922ab8c0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/7e12487971294800990af68c58a0cf91] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.8 K 2024-11-19T08:42:01,954 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8f320f06ae7c401bba9462a7c744ce97, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7a714430fafd488c9df0e078d75f03a0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=38.1 K 2024-11-19T08:42:01,954 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d29f092708f4ca5af9342340f21c08c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732005717573 2024-11-19T08:42:01,954 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f320f06ae7c401bba9462a7c744ce97, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732005717573 2024-11-19T08:42:01,954 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a714430fafd488c9df0e078d75f03a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732005718749 2024-11-19T08:42:01,955 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bacb4f3eca44f9ebc56c92e2c7d3d4d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005720314 2024-11-19T08:42:01,955 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 129578408ca747cbb1a05d95922ab8c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732005718749 2024-11-19T08:42:01,955 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e12487971294800990af68c58a0cf91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005720314 2024-11-19T08:42:01,962 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:01,962 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:01,963 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/c07d531a20db4d089875c073aec9c721 is 50, key is test_row_0/A:col10/1732005720350/Put/seqid=0 2024-11-19T08:42:01,963 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/c6e07d22bac54c0cb72e8298a95a6eb2 is 50, key is test_row_0/B:col10/1732005720350/Put/seqid=0 2024-11-19T08:42:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742278_1454 (size=12459) 2024-11-19T08:42:01,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742279_1455 (size=12459) 2024-11-19T08:42:01,995 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/c6e07d22bac54c0cb72e8298a95a6eb2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/c6e07d22bac54c0cb72e8298a95a6eb2 2024-11-19T08:42:02,005 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into c6e07d22bac54c0cb72e8298a95a6eb2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:02,005 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:02,005 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=13, startTime=1732005721952; duration=0sec 2024-11-19T08:42:02,005 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:02,005 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:42:02,005 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:02,006 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:02,006 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:42:02,006 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:02,006 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e3731fe6b3a940a884751491c74af758, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5518731d355a487191bb6ca3e2204d62, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c4ea581fbcea4162a83177476053bacd] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.8 K 2024-11-19T08:42:02,007 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting e3731fe6b3a940a884751491c74af758, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732005717573 2024-11-19T08:42:02,007 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5518731d355a487191bb6ca3e2204d62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732005718749 2024-11-19T08:42:02,007 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c4ea581fbcea4162a83177476053bacd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005720314 2024-11-19T08:42:02,012 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#386 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:02,012 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/c13acdda57204747b16c058c7a819902 is 50, key is test_row_0/C:col10/1732005720350/Put/seqid=0 2024-11-19T08:42:02,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742280_1456 (size=12459) 2024-11-19T08:42:02,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T08:42:02,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:02,079 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T08:42:02,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/113a734e3bf04236ab70d71fadf028c2 is 50, key is test_row_0/A:col10/1732005721587/Put/seqid=0 2024-11-19T08:42:02,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742281_1457 (size=12151) 2024-11-19T08:42:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T08:42:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005782353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:02,394 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/c07d531a20db4d089875c073aec9c721 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/c07d531a20db4d089875c073aec9c721 2024-11-19T08:42:02,398 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into c07d531a20db4d089875c073aec9c721(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:02,399 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:02,399 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=13, startTime=1732005721952; duration=0sec 2024-11-19T08:42:02,399 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:02,399 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:42:02,418 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/c13acdda57204747b16c058c7a819902 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c13acdda57204747b16c058c7a819902 2024-11-19T08:42:02,421 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into c13acdda57204747b16c058c7a819902(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:02,421 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:02,421 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=13, startTime=1732005721952; duration=0sec 2024-11-19T08:42:02,422 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:02,422 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:42:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005782466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:02,486 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/113a734e3bf04236ab70d71fadf028c2 2024-11-19T08:42:02,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/833c84a2228a472cbb4f7e05cd9c9986 is 50, key is test_row_0/B:col10/1732005721587/Put/seqid=0 2024-11-19T08:42:02,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742282_1458 (size=12151) 2024-11-19T08:42:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T08:42:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005782676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:02,899 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/833c84a2228a472cbb4f7e05cd9c9986 2024-11-19T08:42:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/514b9aa6d2a94cbf8d1ec31f45aeabb7 is 50, key is test_row_0/C:col10/1732005721587/Put/seqid=0 2024-11-19T08:42:02,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742283_1459 (size=12151) 2024-11-19T08:42:02,908 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/514b9aa6d2a94cbf8d1ec31f45aeabb7 2024-11-19T08:42:02,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/113a734e3bf04236ab70d71fadf028c2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/113a734e3bf04236ab70d71fadf028c2 2024-11-19T08:42:02,914 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/113a734e3bf04236ab70d71fadf028c2, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T08:42:02,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/833c84a2228a472cbb4f7e05cd9c9986 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/833c84a2228a472cbb4f7e05cd9c9986 2024-11-19T08:42:02,917 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/833c84a2228a472cbb4f7e05cd9c9986, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T08:42:02,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/514b9aa6d2a94cbf8d1ec31f45aeabb7 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/514b9aa6d2a94cbf8d1ec31f45aeabb7 2024-11-19T08:42:02,927 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/514b9aa6d2a94cbf8d1ec31f45aeabb7, entries=150, sequenceid=230, filesize=11.9 K 2024-11-19T08:42:02,928 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 848ms, sequenceid=230, compaction requested=false 2024-11-19T08:42:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-19T08:42:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-19T08:42:02,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T08:42:02,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-11-19T08:42:02,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4650 sec 2024-11-19T08:42:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:02,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T08:42:02,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/0b5670426d304741b268387acb796a79 is 50, key is test_row_0/A:col10/1732005722352/Put/seqid=0 2024-11-19T08:42:02,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742284_1460 (size=14541) 2024-11-19T08:42:03,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005782995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005782996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,009 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005783002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005783009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005783116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005783126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005783126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005783326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005783336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005783336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/0b5670426d304741b268387acb796a79 2024-11-19T08:42:03,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/8444ddfd09b34b8ea3ded179e149042a is 50, key is test_row_0/B:col10/1732005722352/Put/seqid=0 2024-11-19T08:42:03,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742285_1461 (size=12151) 2024-11-19T08:42:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T08:42:03,626 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:03,627 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-19T08:42:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T08:42:03,628 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:03,629 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:03,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50216 deadline: 1732005783635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005783635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,642 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005783646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:03,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005783646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T08:42:03,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/8444ddfd09b34b8ea3ded179e149042a 2024-11-19T08:42:03,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/fc60ca21bdc64d44a02808514c6881f0 is 50, key is test_row_0/C:col10/1732005722352/Put/seqid=0 2024-11-19T08:42:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742286_1462 (size=12151) 2024-11-19T08:42:03,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T08:42:03,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:03,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:03,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T08:42:03,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:03,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:03,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:03,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T08:42:04,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T08:42:04,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:04,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005784156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005784166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005784166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:04,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/fc60ca21bdc64d44a02808514c6881f0 2024-11-19T08:42:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/0b5670426d304741b268387acb796a79 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/0b5670426d304741b268387acb796a79 2024-11-19T08:42:04,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/0b5670426d304741b268387acb796a79, entries=200, sequenceid=252, filesize=14.2 K 2024-11-19T08:42:04,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/8444ddfd09b34b8ea3ded179e149042a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/8444ddfd09b34b8ea3ded179e149042a 2024-11-19T08:42:04,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/8444ddfd09b34b8ea3ded179e149042a, entries=150, sequenceid=252, filesize=11.9 K 2024-11-19T08:42:04,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/fc60ca21bdc64d44a02808514c6881f0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fc60ca21bdc64d44a02808514c6881f0 2024-11-19T08:42:04,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fc60ca21bdc64d44a02808514c6881f0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-19T08:42:04,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1238ms, sequenceid=252, compaction requested=true 2024-11-19T08:42:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:04,187 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:04,187 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:42:04,188 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:04,188 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:04,188 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/c6e07d22bac54c0cb72e8298a95a6eb2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/833c84a2228a472cbb4f7e05cd9c9986, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/8444ddfd09b34b8ea3ded179e149042a] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.9 K 2024-11-19T08:42:04,188 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/c07d531a20db4d089875c073aec9c721, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/113a734e3bf04236ab70d71fadf028c2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/0b5670426d304741b268387acb796a79] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=38.2 K 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e07d22bac54c0cb72e8298a95a6eb2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005720314 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting c07d531a20db4d089875c073aec9c721, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005720314 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113a734e3bf04236ab70d71fadf028c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005721574 2024-11-19T08:42:04,188 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 833c84a2228a472cbb4f7e05cd9c9986, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005721574 2024-11-19T08:42:04,189 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5670426d304741b268387acb796a79, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005722334 2024-11-19T08:42:04,189 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 8444ddfd09b34b8ea3ded179e149042a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005722334 2024-11-19T08:42:04,193 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:04,193 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:04,194 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/a22ebcc33f194c918a827d97d2714fff is 50, key is test_row_0/B:col10/1732005722352/Put/seqid=0 2024-11-19T08:42:04,194 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/37cf6977dfbf49e180c4b4e7357a491e is 50, key is test_row_0/A:col10/1732005722352/Put/seqid=0 2024-11-19T08:42:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742287_1463 (size=12561) 2024-11-19T08:42:04,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742288_1464 (size=12561) 2024-11-19T08:42:04,202 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/a22ebcc33f194c918a827d97d2714fff as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/a22ebcc33f194c918a827d97d2714fff 2024-11-19T08:42:04,207 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into a22ebcc33f194c918a827d97d2714fff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:04,207 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:04,207 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=13, startTime=1732005724187; duration=0sec 2024-11-19T08:42:04,207 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:04,207 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:42:04,207 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:04,208 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:04,208 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:42:04,208 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:04,208 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c13acdda57204747b16c058c7a819902, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/514b9aa6d2a94cbf8d1ec31f45aeabb7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fc60ca21bdc64d44a02808514c6881f0] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=35.9 K 2024-11-19T08:42:04,208 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c13acdda57204747b16c058c7a819902, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005720314 2024-11-19T08:42:04,208 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 514b9aa6d2a94cbf8d1ec31f45aeabb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732005721574 2024-11-19T08:42:04,209 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting fc60ca21bdc64d44a02808514c6881f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005722334 2024-11-19T08:42:04,213 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:04,214 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/2d4b6dabcf17428890d79c70d9a4dfd4 is 50, key is test_row_0/C:col10/1732005722352/Put/seqid=0 2024-11-19T08:42:04,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742289_1465 (size=12561) 2024-11-19T08:42:04,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T08:42:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:04,239 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T08:42:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f is 50, key is test_row_0/A:col10/1732005723001/Put/seqid=0 2024-11-19T08:42:04,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742290_1466 (size=12301) 2024-11-19T08:42:04,245 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f 2024-11-19T08:42:04,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/85b8d3f4a9d24e2aa144e0bc5befc4f6 is 50, key is test_row_0/B:col10/1732005723001/Put/seqid=0 2024-11-19T08:42:04,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742291_1467 (size=12301) 2024-11-19T08:42:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T08:42:04,600 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/37cf6977dfbf49e180c4b4e7357a491e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37cf6977dfbf49e180c4b4e7357a491e 2024-11-19T08:42:04,604 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 37cf6977dfbf49e180c4b4e7357a491e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:04,604 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:04,604 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=13, startTime=1732005724187; duration=0sec 2024-11-19T08:42:04,604 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:04,604 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:42:04,619 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/2d4b6dabcf17428890d79c70d9a4dfd4 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/2d4b6dabcf17428890d79c70d9a4dfd4 2024-11-19T08:42:04,622 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 2d4b6dabcf17428890d79c70d9a4dfd4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:04,622 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:04,622 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=13, startTime=1732005724187; duration=0sec 2024-11-19T08:42:04,622 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:04,622 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:42:04,653 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/85b8d3f4a9d24e2aa144e0bc5befc4f6 2024-11-19T08:42:04,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/68db37c9c9ce446a9e15633e320650a3 is 50, key is test_row_0/C:col10/1732005723001/Put/seqid=0 2024-11-19T08:42:04,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742292_1468 (size=12301) 2024-11-19T08:42:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T08:42:05,062 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/68db37c9c9ce446a9e15633e320650a3 2024-11-19T08:42:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f 2024-11-19T08:42:05,068 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T08:42:05,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/85b8d3f4a9d24e2aa144e0bc5befc4f6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/85b8d3f4a9d24e2aa144e0bc5befc4f6 2024-11-19T08:42:05,071 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/85b8d3f4a9d24e2aa144e0bc5befc4f6, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T08:42:05,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/68db37c9c9ce446a9e15633e320650a3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/68db37c9c9ce446a9e15633e320650a3 2024-11-19T08:42:05,075 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/68db37c9c9ce446a9e15633e320650a3, entries=150, sequenceid=269, filesize=12.0 K 2024-11-19T08:42:05,075 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 836ms, sequenceid=269, compaction requested=false 2024-11-19T08:42:05,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:05,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:05,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-19T08:42:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-19T08:42:05,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T08:42:05,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-11-19T08:42:05,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4520 sec 2024-11-19T08:42:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:05,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:42:05,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/224b1425325a4cfe9b5d3c93e3f3235b is 50, key is test_row_0/A:col10/1732005725177/Put/seqid=0 2024-11-19T08:42:05,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742293_1469 (size=14741) 2024-11-19T08:42:05,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005785297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005785298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005785299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005785406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005785416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005785416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/224b1425325a4cfe9b5d3c93e3f3235b 2024-11-19T08:42:05,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1e64ae959a61453cbdeb10c494f23d45 is 50, key is test_row_0/B:col10/1732005725177/Put/seqid=0 2024-11-19T08:42:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742294_1470 (size=12301) 2024-11-19T08:42:05,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005785616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005785626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005785626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T08:42:05,776 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:05,776 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-19T08:42:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T08:42:05,778 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:05,778 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:05,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T08:42:05,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:05,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:05,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:05,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005785926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005785946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005785946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1e64ae959a61453cbdeb10c494f23d45 2024-11-19T08:42:06,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/ec2a820fd63f472c85c80fe903c7cc14 is 50, key is test_row_0/C:col10/1732005725177/Put/seqid=0 2024-11-19T08:42:06,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742295_1471 (size=12301) 2024-11-19T08:42:06,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T08:42:06,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:06,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:06,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T08:42:06,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/ec2a820fd63f472c85c80fe903c7cc14 2024-11-19T08:42:06,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/224b1425325a4cfe9b5d3c93e3f3235b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/224b1425325a4cfe9b5d3c93e3f3235b 2024-11-19T08:42:06,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/224b1425325a4cfe9b5d3c93e3f3235b, entries=200, sequenceid=282, filesize=14.4 K 2024-11-19T08:42:06,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1e64ae959a61453cbdeb10c494f23d45 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1e64ae959a61453cbdeb10c494f23d45 2024-11-19T08:42:06,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1e64ae959a61453cbdeb10c494f23d45, entries=150, sequenceid=282, filesize=12.0 K 2024-11-19T08:42:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/ec2a820fd63f472c85c80fe903c7cc14 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ec2a820fd63f472c85c80fe903c7cc14 2024-11-19T08:42:06,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ec2a820fd63f472c85c80fe903c7cc14, entries=150, sequenceid=282, filesize=12.0 K 2024-11-19T08:42:06,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1237ms, sequenceid=282, compaction requested=true 2024-11-19T08:42:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:06,425 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:06,425 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:06,426 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:06,426 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:06,426 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:42:06,426 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:42:06,426 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,426 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,426 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37cf6977dfbf49e180c4b4e7357a491e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/224b1425325a4cfe9b5d3c93e3f3235b] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=38.7 K 2024-11-19T08:42:06,426 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/a22ebcc33f194c918a827d97d2714fff, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/85b8d3f4a9d24e2aa144e0bc5befc4f6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1e64ae959a61453cbdeb10c494f23d45] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=36.3 K 2024-11-19T08:42:06,426 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37cf6977dfbf49e180c4b4e7357a491e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005722334 2024-11-19T08:42:06,426 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a22ebcc33f194c918a827d97d2714fff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005722334 2024-11-19T08:42:06,427 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5c5e2e8183a4fcbaedabcf7bd3bfa6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732005722993 2024-11-19T08:42:06,427 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 85b8d3f4a9d24e2aa144e0bc5befc4f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732005722993 2024-11-19T08:42:06,427 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 224b1425325a4cfe9b5d3c93e3f3235b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732005725177 2024-11-19T08:42:06,427 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e64ae959a61453cbdeb10c494f23d45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732005725177 2024-11-19T08:42:06,431 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:06,432 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/faa683169ab04545bbf281a59b32e3b4 is 50, key is test_row_0/B:col10/1732005725177/Put/seqid=0 2024-11-19T08:42:06,433 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:06,434 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/b6a734faac6b43c18f7ece87dace3ddb is 50, key is test_row_0/A:col10/1732005725177/Put/seqid=0 2024-11-19T08:42:06,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742296_1472 (size=12813) 2024-11-19T08:42:06,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742297_1473 (size=12813) 2024-11-19T08:42:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:06,454 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:06,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T08:42:06,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/37ab1565488f4b9c9d44ac3db60351b3 is 50, key is test_row_0/A:col10/1732005725298/Put/seqid=0 2024-11-19T08:42:06,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742298_1474 (size=14741) 2024-11-19T08:42:06,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005786474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005786476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005786484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005786586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005786596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005786596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:06,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:06,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005786796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005786806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005786806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:06,842 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/faa683169ab04545bbf281a59b32e3b4 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/faa683169ab04545bbf281a59b32e3b4 2024-11-19T08:42:06,843 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/b6a734faac6b43c18f7ece87dace3ddb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/b6a734faac6b43c18f7ece87dace3ddb 2024-11-19T08:42:06,846 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into faa683169ab04545bbf281a59b32e3b4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:06,846 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:06,846 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=13, startTime=1732005726425; duration=0sec 2024-11-19T08:42:06,846 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:06,846 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:42:06,846 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:06,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:06,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:06,850 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:06,850 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:42:06,850 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:06,850 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/2d4b6dabcf17428890d79c70d9a4dfd4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/68db37c9c9ce446a9e15633e320650a3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ec2a820fd63f472c85c80fe903c7cc14] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=36.3 K 2024-11-19T08:42:06,850 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4b6dabcf17428890d79c70d9a4dfd4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732005722334 2024-11-19T08:42:06,850 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into b6a734faac6b43c18f7ece87dace3ddb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:06,851 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:06,851 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=13, startTime=1732005726425; duration=0sec 2024-11-19T08:42:06,851 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:06,851 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:42:06,851 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 68db37c9c9ce446a9e15633e320650a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732005722993 2024-11-19T08:42:06,851 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2a820fd63f472c85c80fe903c7cc14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732005725177 2024-11-19T08:42:06,857 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:06,857 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/b04892388bf8418fa00dd9cf7dc100bf is 50, key is test_row_0/C:col10/1732005725177/Put/seqid=0 2024-11-19T08:42:06,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742299_1475 (size=12813) 2024-11-19T08:42:06,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/37ab1565488f4b9c9d44ac3db60351b3 2024-11-19T08:42:06,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/12639294de5247d3b244abca31ea916b is 50, key is test_row_0/B:col10/1732005725298/Put/seqid=0 2024-11-19T08:42:06,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742300_1476 (size=12301) 2024-11-19T08:42:06,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T08:42:07,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:07,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:07,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50228 deadline: 1732005787036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:07,042 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in e77bbd8dcbf5692aa28a4e1a82d4cbfe of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:07,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005787105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:07,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005787116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005787126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:07,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,267 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/b04892388bf8418fa00dd9cf7dc100bf as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b04892388bf8418fa00dd9cf7dc100bf 2024-11-19T08:42:07,271 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into b04892388bf8418fa00dd9cf7dc100bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:07,271 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:07,271 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=13, startTime=1732005726425; duration=0sec 2024-11-19T08:42:07,271 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:07,271 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:42:07,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/12639294de5247d3b244abca31ea916b 2024-11-19T08:42:07,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/1800512556b84b60be1f2fe4f76eedb2 is 50, key is test_row_0/C:col10/1732005725298/Put/seqid=0 2024-11-19T08:42:07,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742301_1477 (size=12301) 2024-11-19T08:42:07,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:07,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:07,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005787626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:07,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005787626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:07,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005787646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:07,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/1800512556b84b60be1f2fe4f76eedb2 2024-11-19T08:42:07,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/37ab1565488f4b9c9d44ac3db60351b3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37ab1565488f4b9c9d44ac3db60351b3 2024-11-19T08:42:07,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37ab1565488f4b9c9d44ac3db60351b3, entries=200, sequenceid=308, filesize=14.4 K 2024-11-19T08:42:07,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/12639294de5247d3b244abca31ea916b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/12639294de5247d3b244abca31ea916b 2024-11-19T08:42:07,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/12639294de5247d3b244abca31ea916b, entries=150, sequenceid=308, filesize=12.0 K 2024-11-19T08:42:07,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/1800512556b84b60be1f2fe4f76eedb2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/1800512556b84b60be1f2fe4f76eedb2 2024-11-19T08:42:07,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/1800512556b84b60be1f2fe4f76eedb2, entries=150, sequenceid=308, filesize=12.0 K 2024-11-19T08:42:07,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1247ms, sequenceid=308, compaction requested=false 2024-11-19T08:42:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:07,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T08:42:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:07,764 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T08:42:07,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/d2aa10419308430fb0936baa3e72864b is 50, key is test_row_0/A:col10/1732005726483/Put/seqid=0 2024-11-19T08:42:07,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742302_1478 (size=12301) 2024-11-19T08:42:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T08:42:08,170 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/d2aa10419308430fb0936baa3e72864b 2024-11-19T08:42:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1b5e661af3784acebd2d9b2535238ab3 is 50, key is test_row_0/B:col10/1732005726483/Put/seqid=0 2024-11-19T08:42:08,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742303_1479 (size=12301) 2024-11-19T08:42:08,579 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1b5e661af3784acebd2d9b2535238ab3 2024-11-19T08:42:08,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/04e8edda49694edf8ce7a8d510a09ac9 is 50, key is test_row_0/C:col10/1732005726483/Put/seqid=0 2024-11-19T08:42:08,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742304_1480 (size=12301) 2024-11-19T08:42:08,588 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/04e8edda49694edf8ce7a8d510a09ac9 2024-11-19T08:42:08,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/d2aa10419308430fb0936baa3e72864b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/d2aa10419308430fb0936baa3e72864b 2024-11-19T08:42:08,594 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/d2aa10419308430fb0936baa3e72864b, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T08:42:08,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/1b5e661af3784acebd2d9b2535238ab3 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1b5e661af3784acebd2d9b2535238ab3 2024-11-19T08:42:08,598 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1b5e661af3784acebd2d9b2535238ab3, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T08:42:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/04e8edda49694edf8ce7a8d510a09ac9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/04e8edda49694edf8ce7a8d510a09ac9 2024-11-19T08:42:08,604 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/04e8edda49694edf8ce7a8d510a09ac9, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T08:42:08,604 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 841ms, sequenceid=321, compaction requested=true 2024-11-19T08:42:08,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:08,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:08,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-19T08:42:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-19T08:42:08,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T08:42:08,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8270 sec 2024-11-19T08:42:08,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 2.8320 sec 2024-11-19T08:42:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:08,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:08,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:42:08,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/253a76db211744ea8196771eeee1025f is 50, key is test_row_0/A:col10/1732005728660/Put/seqid=0 2024-11-19T08:42:08,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742305_1481 (size=17181) 2024-11-19T08:42:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005788719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:08,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005788720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:08,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005788720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:08,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005788826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:08,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005788836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005788836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:08,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:08,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005789035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005789046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005789046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/253a76db211744ea8196771eeee1025f 2024-11-19T08:42:09,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6ed26bb630c14c51a64783fd4f1aca8d is 50, key is test_row_0/B:col10/1732005728660/Put/seqid=0 2024-11-19T08:42:09,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742306_1482 (size=12301) 2024-11-19T08:42:09,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6ed26bb630c14c51a64783fd4f1aca8d 2024-11-19T08:42:09,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/335e5a3f98f14095b87df904bf22382a is 50, key is test_row_0/C:col10/1732005728660/Put/seqid=0 2024-11-19T08:42:09,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742307_1483 (size=12301) 2024-11-19T08:42:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005789346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005789356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005789366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/335e5a3f98f14095b87df904bf22382a 2024-11-19T08:42:09,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/253a76db211744ea8196771eeee1025f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/253a76db211744ea8196771eeee1025f 2024-11-19T08:42:09,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/253a76db211744ea8196771eeee1025f, entries=250, sequenceid=333, filesize=16.8 K 2024-11-19T08:42:09,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/6ed26bb630c14c51a64783fd4f1aca8d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6ed26bb630c14c51a64783fd4f1aca8d 2024-11-19T08:42:09,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6ed26bb630c14c51a64783fd4f1aca8d, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T08:42:09,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/335e5a3f98f14095b87df904bf22382a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/335e5a3f98f14095b87df904bf22382a 2024-11-19T08:42:09,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/335e5a3f98f14095b87df904bf22382a, entries=150, sequenceid=333, filesize=12.0 K 2024-11-19T08:42:09,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 839ms, sequenceid=333, compaction requested=true 2024-11-19T08:42:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:09,501 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:42:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:09,501 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:42:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e77bbd8dcbf5692aa28a4e1a82d4cbfe:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:09,501 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:42:09,501 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57036 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/B is initiating minor compaction (all files) 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/A is initiating minor compaction (all files) 2024-11-19T08:42:09,502 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/B in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:09,502 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/A in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:09,502 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/faa683169ab04545bbf281a59b32e3b4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/12639294de5247d3b244abca31ea916b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1b5e661af3784acebd2d9b2535238ab3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6ed26bb630c14c51a64783fd4f1aca8d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=48.6 K 2024-11-19T08:42:09,502 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/b6a734faac6b43c18f7ece87dace3ddb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37ab1565488f4b9c9d44ac3db60351b3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/d2aa10419308430fb0936baa3e72864b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/253a76db211744ea8196771eeee1025f] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=55.7 K 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting faa683169ab04545bbf281a59b32e3b4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732005725177 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6a734faac6b43c18f7ece87dace3ddb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732005725177 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 12639294de5247d3b244abca31ea916b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732005725296 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37ab1565488f4b9c9d44ac3db60351b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732005725266 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b5e661af3784acebd2d9b2535238ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732005726473 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2aa10419308430fb0936baa3e72864b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732005726473 2024-11-19T08:42:09,502 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed26bb630c14c51a64783fd4f1aca8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732005728658 2024-11-19T08:42:09,503 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 253a76db211744ea8196771eeee1025f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732005728647 2024-11-19T08:42:09,510 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:09,510 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:09,510 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/b321f541c0c741b29890fd493b640d4d is 50, key is test_row_0/B:col10/1732005728660/Put/seqid=0 2024-11-19T08:42:09,510 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/5bbd979e05fd48899de96c7097fceb5f is 50, key is test_row_0/A:col10/1732005728660/Put/seqid=0 2024-11-19T08:42:09,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742308_1484 (size=12949) 2024-11-19T08:42:09,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742309_1485 (size=12949) 2024-11-19T08:42:09,517 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/b321f541c0c741b29890fd493b640d4d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b321f541c0c741b29890fd493b640d4d 2024-11-19T08:42:09,521 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/B of e77bbd8dcbf5692aa28a4e1a82d4cbfe into b321f541c0c741b29890fd493b640d4d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:09,521 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:09,521 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, priority=12, startTime=1732005729501; duration=0sec 2024-11-19T08:42:09,521 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:09,521 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:B 2024-11-19T08:42:09,521 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:42:09,522 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:42:09,522 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): e77bbd8dcbf5692aa28a4e1a82d4cbfe/C is initiating minor compaction (all files) 2024-11-19T08:42:09,522 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e77bbd8dcbf5692aa28a4e1a82d4cbfe/C in TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:09,522 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b04892388bf8418fa00dd9cf7dc100bf, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/1800512556b84b60be1f2fe4f76eedb2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/04e8edda49694edf8ce7a8d510a09ac9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/335e5a3f98f14095b87df904bf22382a] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp, totalSize=48.6 K 2024-11-19T08:42:09,524 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b04892388bf8418fa00dd9cf7dc100bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732005725177 2024-11-19T08:42:09,525 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 1800512556b84b60be1f2fe4f76eedb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732005725296 2024-11-19T08:42:09,525 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e8edda49694edf8ce7a8d510a09ac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732005726473 2024-11-19T08:42:09,525 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 335e5a3f98f14095b87df904bf22382a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732005728658 2024-11-19T08:42:09,532 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e77bbd8dcbf5692aa28a4e1a82d4cbfe#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:09,532 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/57ab38c78be74fddba141d3fdcb775cc is 50, key is test_row_0/C:col10/1732005728660/Put/seqid=0 2024-11-19T08:42:09,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742310_1486 (size=12949) 2024-11-19T08:42:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:09,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:09,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T08:42:09,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/638b84afa9ef45e989aa388f187ea49c is 50, key is test_row_0/A:col10/1732005729861/Put/seqid=0 2024-11-19T08:42:09,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742311_1487 (size=14741) 2024-11-19T08:42:09,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/638b84afa9ef45e989aa388f187ea49c 2024-11-19T08:42:09,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/4875b99548154ef7b6cfd66323d434c0 is 50, key is test_row_0/B:col10/1732005729861/Put/seqid=0 2024-11-19T08:42:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742312_1488 (size=12301) 2024-11-19T08:42:09,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005789891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005789894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005789895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:09,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:09,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:09,917 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/5bbd979e05fd48899de96c7097fceb5f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5bbd979e05fd48899de96c7097fceb5f 2024-11-19T08:42:09,925 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/A of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 5bbd979e05fd48899de96c7097fceb5f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:09,925 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:09,925 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, priority=12, startTime=1732005729501; duration=0sec 2024-11-19T08:42:09,925 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:09,925 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:A 2024-11-19T08:42:09,945 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/57ab38c78be74fddba141d3fdcb775cc as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/57ab38c78be74fddba141d3fdcb775cc 2024-11-19T08:42:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T08:42:09,946 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:09,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-19T08:42:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T08:42:09,948 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:09,948 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e77bbd8dcbf5692aa28a4e1a82d4cbfe/C of e77bbd8dcbf5692aa28a4e1a82d4cbfe into 57ab38c78be74fddba141d3fdcb775cc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:09,948 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:09,948 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., storeName=e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, priority=12, startTime=1732005729501; duration=0sec 2024-11-19T08:42:09,949 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:09,949 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e77bbd8dcbf5692aa28a4e1a82d4cbfe:C 2024-11-19T08:42:09,949 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:09,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:10,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005790006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005790006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005790016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T08:42:10,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T08:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005790226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005790226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005790226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T08:42:10,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:10,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T08:42:10,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/4875b99548154ef7b6cfd66323d434c0 2024-11-19T08:42:10,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/fe7cb63b6f1e4878a642a4a207f6544c is 50, key is test_row_0/C:col10/1732005729861/Put/seqid=0 2024-11-19T08:42:10,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742313_1489 (size=12301) 2024-11-19T08:42:10,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T08:42:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50200 deadline: 1732005790536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50184 deadline: 1732005790536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50234 deadline: 1732005790536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 , the old value is region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., hostname=3ab37fa97a98,34165,1732005593269, seqNum=2 because the exception is null or not the one we care about 2024-11-19T08:42:10,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T08:42:10,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:10,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T08:42:10,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/fe7cb63b6f1e4878a642a4a207f6544c 2024-11-19T08:42:10,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/638b84afa9ef45e989aa388f187ea49c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/638b84afa9ef45e989aa388f187ea49c 2024-11-19T08:42:10,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/638b84afa9ef45e989aa388f187ea49c, entries=200, sequenceid=359, filesize=14.4 K 2024-11-19T08:42:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/4875b99548154ef7b6cfd66323d434c0 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4875b99548154ef7b6cfd66323d434c0 2024-11-19T08:42:10,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T08:42:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:10,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4875b99548154ef7b6cfd66323d434c0, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T08:42:10,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/fe7cb63b6f1e4878a642a4a207f6544c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fe7cb63b6f1e4878a642a4a207f6544c 2024-11-19T08:42:10,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fe7cb63b6f1e4878a642a4a207f6544c, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T08:42:10,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 856ms, sequenceid=359, compaction requested=false 2024-11-19T08:42:10,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:10,731 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-19T08:42:10,731 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:10,731 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,731 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:10,733 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-19T08:42:10,733 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:10,733 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,733 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:10,736 INFO [Thread-1857 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1857. 2024-11-19T08:42:10,736 DEBUG [Thread-1857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:10,736 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,736 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:10,736 INFO [Thread-1855 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1855. 2024-11-19T08:42:10,736 DEBUG [Thread-1855 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:10,736 DEBUG [Thread-1855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,736 DEBUG [Thread-1855 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:10,738 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-11-19T08:42:10,738 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:10,738 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,738 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:10,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:10,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T08:42:10,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:10,866 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T08:42:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/e1dc59882f5b4fd5ba28935c24f3391c is 50, key is test_row_0/A:col10/1732005729894/Put/seqid=0 2024-11-19T08:42:10,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742314_1490 (size=12301) 2024-11-19T08:42:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:11,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. as already flushing 2024-11-19T08:42:11,050 INFO [Thread-1846 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1846. 2024-11-19T08:42:11,050 DEBUG [Thread-1846 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:11,050 DEBUG [Thread-1846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:11,051 DEBUG [Thread-1846 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:11,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:11,060 INFO [Thread-1844 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1844. 2024-11-19T08:42:11,060 INFO [Thread-1852 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1852. 2024-11-19T08:42:11,060 DEBUG [Thread-1852 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:11,060 DEBUG [Thread-1844 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:11,060 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:11,060 DEBUG [Thread-1844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:11,060 DEBUG [Thread-1844 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:11,060 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:11,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:11,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T08:42:11,281 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/e1dc59882f5b4fd5ba28935c24f3391c 2024-11-19T08:42:11,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/67c59627c56d466c9051ce7ad833c3d9 is 50, key is test_row_0/B:col10/1732005729894/Put/seqid=0 2024-11-19T08:42:11,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742315_1491 (size=12301) 2024-11-19T08:42:11,701 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/67c59627c56d466c9051ce7ad833c3d9 2024-11-19T08:42:11,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/ef53a6fc8b55438fae331d653e2ad586 is 50, key is test_row_0/C:col10/1732005729894/Put/seqid=0 2024-11-19T08:42:11,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742316_1492 (size=12301) 2024-11-19T08:42:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T08:42:12,119 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/ef53a6fc8b55438fae331d653e2ad586 2024-11-19T08:42:12,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/e1dc59882f5b4fd5ba28935c24f3391c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e1dc59882f5b4fd5ba28935c24f3391c 2024-11-19T08:42:12,132 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e1dc59882f5b4fd5ba28935c24f3391c, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T08:42:12,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/67c59627c56d466c9051ce7ad833c3d9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/67c59627c56d466c9051ce7ad833c3d9 2024-11-19T08:42:12,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/67c59627c56d466c9051ce7ad833c3d9, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T08:42:12,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/ef53a6fc8b55438fae331d653e2ad586 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ef53a6fc8b55438fae331d653e2ad586 2024-11-19T08:42:12,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ef53a6fc8b55438fae331d653e2ad586, entries=150, sequenceid=372, filesize=12.0 K 2024-11-19T08:42:12,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=20.13 KB/20610 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1280ms, sequenceid=372, compaction requested=true 2024-11-19T08:42:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: 2024-11-19T08:42:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-19T08:42:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-19T08:42:12,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T08:42:12,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1980 sec 2024-11-19T08:42:12,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 2.2030 sec 2024-11-19T08:42:13,727 INFO [Thread-1850 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1850. 2024-11-19T08:42:13,727 DEBUG [Thread-1850 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:13,727 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:13,727 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:13,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T08:42:14,108 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:17,131 INFO [Thread-1848 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1848. 2024-11-19T08:42:17,131 DEBUG [Thread-1848 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:17,131 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:17,131 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:17,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1502 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4500 rows 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1495 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4483 rows 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1488 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4462 rows 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1489 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4459 rows 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1494 2024-11-19T08:42:17,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4479 rows 2024-11-19T08:42:17,132 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:42:17,132 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:42:17,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:17,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:17,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:17,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T08:42:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T08:42:17,139 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005737139"}]},"ts":"1732005737139"} 2024-11-19T08:42:17,142 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T08:42:17,142 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T08:42:17,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:42:17,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, UNASSIGN}] 2024-11-19T08:42:17,144 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, UNASSIGN 2024-11-19T08:42:17,145 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e77bbd8dcbf5692aa28a4e1a82d4cbfe, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:17,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, UNASSIGN because future has completed 2024-11-19T08:42:17,146 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:42:17,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:42:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T08:42:17,300 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:17,300 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:42:17,300 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing e77bbd8dcbf5692aa28a4e1a82d4cbfe, disabling compactions & flushes 2024-11-19T08:42:17,300 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:17,300 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:17,300 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. after waiting 0 ms 2024-11-19T08:42:17,300 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:17,301 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T08:42:17,308 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/4e21a0be8f4d47aa9b26e960e943cffc is 50, key is test_row_0/A:col10/1732005737127/Put/seqid=0 2024-11-19T08:42:17,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742317_1493 (size=12301) 2024-11-19T08:42:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T08:42:17,714 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/4e21a0be8f4d47aa9b26e960e943cffc 2024-11-19T08:42:17,722 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/e922572ae9ca4336b1329f93a7425015 is 50, key is test_row_0/B:col10/1732005737127/Put/seqid=0 2024-11-19T08:42:17,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742318_1494 (size=12301) 2024-11-19T08:42:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T08:42:18,128 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/e922572ae9ca4336b1329f93a7425015 2024-11-19T08:42:18,140 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/77edc68f7461490592034cbca16325bd is 50, key is test_row_0/C:col10/1732005737127/Put/seqid=0 2024-11-19T08:42:18,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742319_1495 (size=12301) 2024-11-19T08:42:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T08:42:18,546 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/77edc68f7461490592034cbca16325bd 2024-11-19T08:42:18,558 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/A/4e21a0be8f4d47aa9b26e960e943cffc as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/4e21a0be8f4d47aa9b26e960e943cffc 2024-11-19T08:42:18,562 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/4e21a0be8f4d47aa9b26e960e943cffc, entries=150, sequenceid=380, filesize=12.0 K 2024-11-19T08:42:18,563 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/B/e922572ae9ca4336b1329f93a7425015 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/e922572ae9ca4336b1329f93a7425015 2024-11-19T08:42:18,567 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/e922572ae9ca4336b1329f93a7425015, entries=150, sequenceid=380, filesize=12.0 K 2024-11-19T08:42:18,567 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/.tmp/C/77edc68f7461490592034cbca16325bd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/77edc68f7461490592034cbca16325bd 2024-11-19T08:42:18,572 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/77edc68f7461490592034cbca16325bd, entries=150, sequenceid=380, filesize=12.0 K 2024-11-19T08:42:18,573 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1273ms, sequenceid=380, compaction requested=true 2024-11-19T08:42:18,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/67b1a5261da249afaeeaec4e34bede94, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7134dfbd38164d8eb04d769b391745cb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/ab72c14e8c284447ad6b9e0b2807cf49, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5f32f68d4d7c4d109005a3e09c053675, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/3458e8d5f24b4722bb296152b3fb5eee, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5a369fd541f64d938d43b16e92e8e088, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8f320f06ae7c401bba9462a7c744ce97, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7a714430fafd488c9df0e078d75f03a0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/c07d531a20db4d089875c073aec9c721, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/113a734e3bf04236ab70d71fadf028c2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/0b5670426d304741b268387acb796a79, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37cf6977dfbf49e180c4b4e7357a491e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/224b1425325a4cfe9b5d3c93e3f3235b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/b6a734faac6b43c18f7ece87dace3ddb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37ab1565488f4b9c9d44ac3db60351b3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/d2aa10419308430fb0936baa3e72864b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/253a76db211744ea8196771eeee1025f] to archive 2024-11-19T08:42:18,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:42:18,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/67b1a5261da249afaeeaec4e34bede94 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/67b1a5261da249afaeeaec4e34bede94 2024-11-19T08:42:18,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7134dfbd38164d8eb04d769b391745cb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7134dfbd38164d8eb04d769b391745cb 2024-11-19T08:42:18,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/ab72c14e8c284447ad6b9e0b2807cf49 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/ab72c14e8c284447ad6b9e0b2807cf49 2024-11-19T08:42:18,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5f32f68d4d7c4d109005a3e09c053675 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5f32f68d4d7c4d109005a3e09c053675 2024-11-19T08:42:18,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/3458e8d5f24b4722bb296152b3fb5eee to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/3458e8d5f24b4722bb296152b3fb5eee 2024-11-19T08:42:18,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5a369fd541f64d938d43b16e92e8e088 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5a369fd541f64d938d43b16e92e8e088 2024-11-19T08:42:18,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8f320f06ae7c401bba9462a7c744ce97 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/8f320f06ae7c401bba9462a7c744ce97 2024-11-19T08:42:18,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7a714430fafd488c9df0e078d75f03a0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/7a714430fafd488c9df0e078d75f03a0 2024-11-19T08:42:18,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/2bacb4f3eca44f9ebc56c92e2c7d3d4d 2024-11-19T08:42:18,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/c07d531a20db4d089875c073aec9c721 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/c07d531a20db4d089875c073aec9c721 2024-11-19T08:42:18,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/113a734e3bf04236ab70d71fadf028c2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/113a734e3bf04236ab70d71fadf028c2 2024-11-19T08:42:18,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/0b5670426d304741b268387acb796a79 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/0b5670426d304741b268387acb796a79 2024-11-19T08:42:18,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37cf6977dfbf49e180c4b4e7357a491e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37cf6977dfbf49e180c4b4e7357a491e 2024-11-19T08:42:18,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e5c5e2e8183a4fcbaedabcf7bd3bfa6f 2024-11-19T08:42:18,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/224b1425325a4cfe9b5d3c93e3f3235b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/224b1425325a4cfe9b5d3c93e3f3235b 2024-11-19T08:42:18,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/b6a734faac6b43c18f7ece87dace3ddb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/b6a734faac6b43c18f7ece87dace3ddb 2024-11-19T08:42:18,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37ab1565488f4b9c9d44ac3db60351b3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/37ab1565488f4b9c9d44ac3db60351b3 2024-11-19T08:42:18,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/d2aa10419308430fb0936baa3e72864b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/d2aa10419308430fb0936baa3e72864b 2024-11-19T08:42:18,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/253a76db211744ea8196771eeee1025f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/253a76db211744ea8196771eeee1025f 2024-11-19T08:42:18,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/0a2e380f329940e8bd23e0e93806981f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/166aaba028f14756bae28273ddda7614, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1deb73da926e4ca4bf382dd7cd80c808, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6b5e36efa5184986b4ffa91d0dbf0923, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3e500db4a26b49879e24a99cbd49118a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1d29f092708f4ca5af9342340f21c08c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3c76fa1e3e8b41aba23dfeb3d808b07e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/129578408ca747cbb1a05d95922ab8c0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/c6e07d22bac54c0cb72e8298a95a6eb2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/7e12487971294800990af68c58a0cf91, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/833c84a2228a472cbb4f7e05cd9c9986, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/a22ebcc33f194c918a827d97d2714fff, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/8444ddfd09b34b8ea3ded179e149042a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/85b8d3f4a9d24e2aa144e0bc5befc4f6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/faa683169ab04545bbf281a59b32e3b4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1e64ae959a61453cbdeb10c494f23d45, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/12639294de5247d3b244abca31ea916b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1b5e661af3784acebd2d9b2535238ab3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6ed26bb630c14c51a64783fd4f1aca8d] to archive 2024-11-19T08:42:18,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:42:18,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/0a2e380f329940e8bd23e0e93806981f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/0a2e380f329940e8bd23e0e93806981f 2024-11-19T08:42:18,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/166aaba028f14756bae28273ddda7614 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/166aaba028f14756bae28273ddda7614 2024-11-19T08:42:18,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1deb73da926e4ca4bf382dd7cd80c808 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1deb73da926e4ca4bf382dd7cd80c808 2024-11-19T08:42:18,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6b5e36efa5184986b4ffa91d0dbf0923 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6b5e36efa5184986b4ffa91d0dbf0923 2024-11-19T08:42:18,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3e500db4a26b49879e24a99cbd49118a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3e500db4a26b49879e24a99cbd49118a 2024-11-19T08:42:18,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1d29f092708f4ca5af9342340f21c08c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1d29f092708f4ca5af9342340f21c08c 2024-11-19T08:42:18,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3c76fa1e3e8b41aba23dfeb3d808b07e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/3c76fa1e3e8b41aba23dfeb3d808b07e 2024-11-19T08:42:18,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/129578408ca747cbb1a05d95922ab8c0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/129578408ca747cbb1a05d95922ab8c0 2024-11-19T08:42:18,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/c6e07d22bac54c0cb72e8298a95a6eb2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/c6e07d22bac54c0cb72e8298a95a6eb2 2024-11-19T08:42:18,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/7e12487971294800990af68c58a0cf91 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/7e12487971294800990af68c58a0cf91 2024-11-19T08:42:18,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/833c84a2228a472cbb4f7e05cd9c9986 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/833c84a2228a472cbb4f7e05cd9c9986 2024-11-19T08:42:18,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/a22ebcc33f194c918a827d97d2714fff to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/a22ebcc33f194c918a827d97d2714fff 2024-11-19T08:42:18,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/8444ddfd09b34b8ea3ded179e149042a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/8444ddfd09b34b8ea3ded179e149042a 2024-11-19T08:42:18,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/85b8d3f4a9d24e2aa144e0bc5befc4f6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/85b8d3f4a9d24e2aa144e0bc5befc4f6 2024-11-19T08:42:18,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/faa683169ab04545bbf281a59b32e3b4 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/faa683169ab04545bbf281a59b32e3b4 2024-11-19T08:42:18,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1e64ae959a61453cbdeb10c494f23d45 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1e64ae959a61453cbdeb10c494f23d45 2024-11-19T08:42:18,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/12639294de5247d3b244abca31ea916b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/12639294de5247d3b244abca31ea916b 2024-11-19T08:42:18,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1b5e661af3784acebd2d9b2535238ab3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/1b5e661af3784acebd2d9b2535238ab3 2024-11-19T08:42:18,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6ed26bb630c14c51a64783fd4f1aca8d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/6ed26bb630c14c51a64783fd4f1aca8d 2024-11-19T08:42:18,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/a162c8cb560d45a2943f98c8e35d035b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5171c5ed02fd4201872744a6257e0071, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/53dcfb18d5ed497fb291d6163a42c86d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e58ab71d114b46a7bcb7b34a0033834f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3d449c1fedc94cdda7773050f7290735, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e3731fe6b3a940a884751491c74af758, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/06752a737cdc41cdace50dfb24e69b28, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5518731d355a487191bb6ca3e2204d62, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c13acdda57204747b16c058c7a819902, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c4ea581fbcea4162a83177476053bacd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/514b9aa6d2a94cbf8d1ec31f45aeabb7, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/2d4b6dabcf17428890d79c70d9a4dfd4, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fc60ca21bdc64d44a02808514c6881f0, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/68db37c9c9ce446a9e15633e320650a3, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b04892388bf8418fa00dd9cf7dc100bf, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ec2a820fd63f472c85c80fe903c7cc14, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/1800512556b84b60be1f2fe4f76eedb2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/04e8edda49694edf8ce7a8d510a09ac9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/335e5a3f98f14095b87df904bf22382a] to archive 2024-11-19T08:42:18,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:42:18,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/a162c8cb560d45a2943f98c8e35d035b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/a162c8cb560d45a2943f98c8e35d035b 2024-11-19T08:42:18,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5171c5ed02fd4201872744a6257e0071 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5171c5ed02fd4201872744a6257e0071 2024-11-19T08:42:18,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/53dcfb18d5ed497fb291d6163a42c86d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/53dcfb18d5ed497fb291d6163a42c86d 2024-11-19T08:42:18,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e58ab71d114b46a7bcb7b34a0033834f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e58ab71d114b46a7bcb7b34a0033834f 2024-11-19T08:42:18,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3d449c1fedc94cdda7773050f7290735 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/3d449c1fedc94cdda7773050f7290735 2024-11-19T08:42:18,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e3731fe6b3a940a884751491c74af758 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/e3731fe6b3a940a884751491c74af758 2024-11-19T08:42:18,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/06752a737cdc41cdace50dfb24e69b28 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/06752a737cdc41cdace50dfb24e69b28 2024-11-19T08:42:18,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5518731d355a487191bb6ca3e2204d62 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/5518731d355a487191bb6ca3e2204d62 2024-11-19T08:42:18,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c13acdda57204747b16c058c7a819902 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c13acdda57204747b16c058c7a819902 2024-11-19T08:42:18,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c4ea581fbcea4162a83177476053bacd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/c4ea581fbcea4162a83177476053bacd 2024-11-19T08:42:18,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/514b9aa6d2a94cbf8d1ec31f45aeabb7 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/514b9aa6d2a94cbf8d1ec31f45aeabb7 2024-11-19T08:42:18,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/2d4b6dabcf17428890d79c70d9a4dfd4 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/2d4b6dabcf17428890d79c70d9a4dfd4 2024-11-19T08:42:18,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fc60ca21bdc64d44a02808514c6881f0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fc60ca21bdc64d44a02808514c6881f0 2024-11-19T08:42:18,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/68db37c9c9ce446a9e15633e320650a3 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/68db37c9c9ce446a9e15633e320650a3 2024-11-19T08:42:18,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b04892388bf8418fa00dd9cf7dc100bf to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/b04892388bf8418fa00dd9cf7dc100bf 2024-11-19T08:42:18,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ec2a820fd63f472c85c80fe903c7cc14 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ec2a820fd63f472c85c80fe903c7cc14 2024-11-19T08:42:18,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/1800512556b84b60be1f2fe4f76eedb2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/1800512556b84b60be1f2fe4f76eedb2 2024-11-19T08:42:18,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/04e8edda49694edf8ce7a8d510a09ac9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/04e8edda49694edf8ce7a8d510a09ac9 2024-11-19T08:42:18,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/335e5a3f98f14095b87df904bf22382a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/335e5a3f98f14095b87df904bf22382a 2024-11-19T08:42:18,628 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/recovered.edits/383.seqid, newMaxSeqId=383, maxSeqId=1 2024-11-19T08:42:18,628 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. 2024-11-19T08:42:18,628 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for e77bbd8dcbf5692aa28a4e1a82d4cbfe: Waiting for close lock at 1732005737300Running coprocessor pre-close hooks at 1732005737300Disabling compacts and flushes for region at 1732005737300Disabling writes for close at 1732005737300Obtaining lock to block concurrent updates at 1732005737301 (+1 ms)Preparing flush snapshotting stores in e77bbd8dcbf5692aa28a4e1a82d4cbfe at 1732005737301Finished memstore snapshotting TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732005737301Flushing stores of TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe. at 1732005737303 (+2 ms)Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/A: creating writer at 1732005737303Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/A: appending metadata at 1732005737308 (+5 ms)Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/A: closing flushed file at 1732005737308Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/B: creating writer at 1732005737719 (+411 ms)Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/B: appending metadata at 1732005737722 (+3 ms)Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/B: closing flushed file at 1732005737722Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/C: creating writer at 1732005738136 (+414 ms)Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/C: appending metadata at 1732005738140 (+4 ms)Flushing e77bbd8dcbf5692aa28a4e1a82d4cbfe/C: closing flushed file at 1732005738140Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b49562b: reopening flushed file at 1732005738556 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@642e08a5: reopening flushed file at 1732005738562 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5688e015: reopening flushed file at 1732005738567 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e77bbd8dcbf5692aa28a4e1a82d4cbfe in 1273ms, sequenceid=380, compaction requested=true at 1732005738573 (+6 ms)Writing region close event to WAL at 1732005738625 (+52 ms)Running coprocessor post-close hooks at 1732005738628 (+3 ms)Closed at 1732005738628 2024-11-19T08:42:18,630 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:18,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e77bbd8dcbf5692aa28a4e1a82d4cbfe, regionState=CLOSED 2024-11-19T08:42:18,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:42:18,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-19T08:42:18,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure e77bbd8dcbf5692aa28a4e1a82d4cbfe, server=3ab37fa97a98,34165,1732005593269 in 1.4860 sec 2024-11-19T08:42:18,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T08:42:18,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e77bbd8dcbf5692aa28a4e1a82d4cbfe, UNASSIGN in 1.4900 sec 2024-11-19T08:42:18,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-19T08:42:18,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4930 sec 2024-11-19T08:42:18,639 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005738639"}]},"ts":"1732005738639"} 2024-11-19T08:42:18,641 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T08:42:18,641 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T08:42:18,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5070 sec 2024-11-19T08:42:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T08:42:19,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:19,289 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T08:42:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:19,293 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-19T08:42:19,295 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:19,301 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:19,305 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/recovered.edits] 2024-11-19T08:42:19,310 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/4e21a0be8f4d47aa9b26e960e943cffc to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/4e21a0be8f4d47aa9b26e960e943cffc 2024-11-19T08:42:19,311 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5bbd979e05fd48899de96c7097fceb5f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/5bbd979e05fd48899de96c7097fceb5f 2024-11-19T08:42:19,313 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/638b84afa9ef45e989aa388f187ea49c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/638b84afa9ef45e989aa388f187ea49c 2024-11-19T08:42:19,315 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e1dc59882f5b4fd5ba28935c24f3391c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/A/e1dc59882f5b4fd5ba28935c24f3391c 2024-11-19T08:42:19,318 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4875b99548154ef7b6cfd66323d434c0 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/4875b99548154ef7b6cfd66323d434c0 2024-11-19T08:42:19,320 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/67c59627c56d466c9051ce7ad833c3d9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/67c59627c56d466c9051ce7ad833c3d9 2024-11-19T08:42:19,322 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b321f541c0c741b29890fd493b640d4d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/b321f541c0c741b29890fd493b640d4d 2024-11-19T08:42:19,323 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/e922572ae9ca4336b1329f93a7425015 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/B/e922572ae9ca4336b1329f93a7425015 2024-11-19T08:42:19,326 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/57ab38c78be74fddba141d3fdcb775cc to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/57ab38c78be74fddba141d3fdcb775cc 2024-11-19T08:42:19,328 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/77edc68f7461490592034cbca16325bd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/77edc68f7461490592034cbca16325bd 2024-11-19T08:42:19,329 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ef53a6fc8b55438fae331d653e2ad586 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/ef53a6fc8b55438fae331d653e2ad586 2024-11-19T08:42:19,331 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fe7cb63b6f1e4878a642a4a207f6544c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/C/fe7cb63b6f1e4878a642a4a207f6544c 2024-11-19T08:42:19,335 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/recovered.edits/383.seqid to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe/recovered.edits/383.seqid 2024-11-19T08:42:19,335 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/e77bbd8dcbf5692aa28a4e1a82d4cbfe 2024-11-19T08:42:19,336 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T08:42:19,339 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:19,342 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T08:42:19,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T08:42:19,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:19,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T08:42:19,347 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732005739346"}]},"ts":"9223372036854775807"} 2024-11-19T08:42:19,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T08:42:19,349 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e77bbd8dcbf5692aa28a4e1a82d4cbfe, NAME => 'TestAcidGuarantees,,1732005708532.e77bbd8dcbf5692aa28a4e1a82d4cbfe.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T08:42:19,349 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T08:42:19,349 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732005739349"}]},"ts":"9223372036854775807"} 2024-11-19T08:42:19,351 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T08:42:19,351 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:19,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-19T08:42:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-19T08:42:19,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T08:42:19,412 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:19,422 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 242), OpenFileDescriptor=449 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 463), ProcessCount=11 (was 11), AvailableMemoryMB=5786 (was 5818) 2024-11-19T08:42:19,431 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=5786 2024-11-19T08:42:19,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:42:19,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:42:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:19,436 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T08:42:19,436 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:19,436 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-11-19T08:42:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T08:42:19,437 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T08:42:19,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742320_1496 (size=959) 2024-11-19T08:42:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T08:42:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T08:42:19,848 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a 2024-11-19T08:42:19,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742321_1497 (size=53) 2024-11-19T08:42:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T08:42:20,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:42:20,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f0224c56755d95b541219e376c56cd39, disabling compactions & flushes 2024-11-19T08:42:20,260 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:20,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:20,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. after waiting 0 ms 2024-11-19T08:42:20,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:20,260 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:20,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0224c56755d95b541219e376c56cd39: Waiting for close lock at 1732005740260Disabling compacts and flushes for region at 1732005740260Disabling writes for close at 1732005740260Writing region close event to WAL at 1732005740260Closed at 1732005740260 2024-11-19T08:42:20,262 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T08:42:20,263 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732005740263"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732005740263"}]},"ts":"1732005740263"} 2024-11-19T08:42:20,265 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T08:42:20,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T08:42:20,267 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005740266"}]},"ts":"1732005740266"} 2024-11-19T08:42:20,269 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T08:42:20,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, ASSIGN}] 2024-11-19T08:42:20,271 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, ASSIGN 2024-11-19T08:42:20,272 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, ASSIGN; state=OFFLINE, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=false 2024-11-19T08:42:20,423 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:20,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, ASSIGN because future has completed 2024-11-19T08:42:20,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:42:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T08:42:20,585 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:20,586 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:42:20,586 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,587 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:42:20,587 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,587 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,590 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,592 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0224c56755d95b541219e376c56cd39 columnFamilyName A 2024-11-19T08:42:20,593 DEBUG [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:20,593 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(327): Store=f0224c56755d95b541219e376c56cd39/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:42:20,594 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,595 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0224c56755d95b541219e376c56cd39 columnFamilyName B 2024-11-19T08:42:20,595 DEBUG [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:20,596 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(327): Store=f0224c56755d95b541219e376c56cd39/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:42:20,596 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,597 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0224c56755d95b541219e376c56cd39 columnFamilyName C 2024-11-19T08:42:20,597 DEBUG [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:20,598 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(327): Store=f0224c56755d95b541219e376c56cd39/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:42:20,598 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,599 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,599 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,601 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,601 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,601 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:42:20,603 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,606 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T08:42:20,606 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened f0224c56755d95b541219e376c56cd39; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62218468, jitterRate=-0.0728725790977478}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:42:20,606 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:20,607 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for f0224c56755d95b541219e376c56cd39: Running coprocessor pre-open hook at 1732005740587Writing region info on filesystem at 1732005740587Initializing all the Stores at 1732005740589 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005740589Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005740589Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005740589Cleaning up temporary data from old regions at 1732005740601 (+12 ms)Running coprocessor post-open hooks at 1732005740606 (+5 ms)Region opened successfully at 1732005740607 (+1 ms) 2024-11-19T08:42:20,608 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., pid=152, masterSystemTime=1732005740579 2024-11-19T08:42:20,611 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:20,611 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:20,611 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=OPEN, openSeqNum=2, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:20,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:42:20,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-19T08:42:20,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 in 187 msec 2024-11-19T08:42:20,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T08:42:20,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, ASSIGN in 345 msec 2024-11-19T08:42:20,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T08:42:20,616 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005740616"}]},"ts":"1732005740616"} 2024-11-19T08:42:20,617 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T08:42:20,618 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T08:42:20,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-19T08:42:21,525 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T08:42:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T08:42:21,587 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:21,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:21,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:21,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:21,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9f803e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:21,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a37d543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:21,592 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:21,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:21,595 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T08:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3ab37fa97a98,36325,1732005592414 2024-11-19T08:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13e6c07a 2024-11-19T08:42:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T08:42:21,596 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T08:42:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T08:42:21,598 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T08:42:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:21,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742322_1498 (size=995) 2024-11-19T08:42:22,009 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T08:42:22,009 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T08:42:22,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:42:22,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, REOPEN/MOVE}] 2024-11-19T08:42:22,013 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, REOPEN/MOVE 2024-11-19T08:42:22,013 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, REOPEN/MOVE because future has completed 2024-11-19T08:42:22,015 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:42:22,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:42:22,169 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,169 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:42:22,169 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing f0224c56755d95b541219e376c56cd39, disabling compactions & flushes 2024-11-19T08:42:22,169 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,169 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,169 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. after waiting 0 ms 2024-11-19T08:42:22,169 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,176 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T08:42:22,177 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,177 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for f0224c56755d95b541219e376c56cd39: Waiting for close lock at 1732005742169Running coprocessor pre-close hooks at 1732005742169Disabling compacts and flushes for region at 1732005742169Disabling writes for close at 1732005742169Writing region close event to WAL at 1732005742170 (+1 ms)Running coprocessor post-close hooks at 1732005742177 (+7 ms)Closed at 1732005742177 2024-11-19T08:42:22,177 WARN [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: f0224c56755d95b541219e376c56cd39 to self. 2024-11-19T08:42:22,178 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,179 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=CLOSED 2024-11-19T08:42:22,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:42:22,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-19T08:42:22,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 in 166 msec 2024-11-19T08:42:22,183 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, REOPEN/MOVE; state=CLOSED, location=3ab37fa97a98,34165,1732005593269; forceNewPlan=false, retain=true 2024-11-19T08:42:22,334 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=OPENING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, REOPEN/MOVE because future has completed 2024-11-19T08:42:22,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:42:22,495 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,495 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} 2024-11-19T08:42:22,496 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,496 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T08:42:22,496 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,496 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,498 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,499 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0224c56755d95b541219e376c56cd39 columnFamilyName A 2024-11-19T08:42:22,501 DEBUG [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:22,502 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(327): Store=f0224c56755d95b541219e376c56cd39/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:42:22,502 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,503 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0224c56755d95b541219e376c56cd39 columnFamilyName B 2024-11-19T08:42:22,503 DEBUG [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:22,503 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(327): Store=f0224c56755d95b541219e376c56cd39/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:42:22,503 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,503 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0224c56755d95b541219e376c56cd39 columnFamilyName C 2024-11-19T08:42:22,503 DEBUG [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:22,504 INFO [StoreOpener-f0224c56755d95b541219e376c56cd39-1 {}] regionserver.HStore(327): Store=f0224c56755d95b541219e376c56cd39/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T08:42:22,504 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,505 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,505 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,506 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,506 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,506 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T08:42:22,507 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,508 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened f0224c56755d95b541219e376c56cd39; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60793184, jitterRate=-0.09411096572875977}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T08:42:22,508 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,508 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for f0224c56755d95b541219e376c56cd39: Running coprocessor pre-open hook at 1732005742497Writing region info on filesystem at 1732005742497Initializing all the Stores at 1732005742498 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005742498Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005742498Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732005742498Cleaning up temporary data from old regions at 1732005742506 (+8 ms)Running coprocessor post-open hooks at 1732005742508 (+2 ms)Region opened successfully at 1732005742508 2024-11-19T08:42:22,509 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., pid=157, masterSystemTime=1732005742488 2024-11-19T08:42:22,510 DEBUG [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,510 INFO [RS_OPEN_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,511 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=OPEN, openSeqNum=5, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:42:22,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-11-19T08:42:22,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 in 176 msec 2024-11-19T08:42:22,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T08:42:22,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, REOPEN/MOVE in 501 msec 2024-11-19T08:42:22,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-19T08:42:22,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-19T08:42:22,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-11-19T08:42:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-19T08:42:22,519 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:22,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,520 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2731755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,521 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45614972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,522 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4723fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,523 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431a077b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,525 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2087a49f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,526 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11443619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,527 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,528 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49842751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,529 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fecd805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,531 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259d143a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,532 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb6aa4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,534 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb1bb00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,535 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6e3a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,537 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4745b9a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,538 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363e11cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,541 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc88cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,543 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482a4bad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,545 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3113b06f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,546 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3757d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3ab37fa97a98,36325,-1 for getting cluster id 2024-11-19T08:42:22,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T08:42:22,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '80b3abe4-4536-4a59-95a0-4a431c025320' 2024-11-19T08:42:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T08:42:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "80b3abe4-4536-4a59-95a0-4a431c025320" 2024-11-19T08:42:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c19644f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3ab37fa97a98,36325,-1] 2024-11-19T08:42:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T08:42:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:22,550 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T08:42:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394dfd7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T08:42:22,551 DEBUG [Thread-2234 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,551 DEBUG [Thread-2251 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,551 DEBUG [Thread-2249 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,552 DEBUG [Thread-2238 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:22,552 DEBUG [Thread-2232 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,552 DEBUG [Thread-2243 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,553 DEBUG [Thread-2245 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,553 DEBUG [Thread-2240 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,553 DEBUG [Thread-2247 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-19T08:42:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,556 DEBUG [Thread-2236 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T08:42:22,556 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T08:42:22,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3ab37fa97a98,34165,1732005593269, seqNum=-1] 2024-11-19T08:42:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T08:42:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:22,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T08:42:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5] 2024-11-19T08:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:22,567 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:22,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T08:42:22,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005802584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005802585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005802585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191e64c3019045460bb7e3637d500cb782_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005742566/Put/seqid=0 2024-11-19T08:42:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005802588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005802589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742323_1499 (size=12154) 2024-11-19T08:42:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T08:42:22,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005802696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005802696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005802696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005802697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005802697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:22,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:22,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:22,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T08:42:22,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005802906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005802905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005802906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005802906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005802906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:23,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191e64c3019045460bb7e3637d500cb782_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191e64c3019045460bb7e3637d500cb782_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:23,014 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/af2a539d9c594f39be75e749b6a081da, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:23,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/af2a539d9c594f39be75e749b6a081da is 175, key is test_row_0/A:col10/1732005742566/Put/seqid=0 2024-11-19T08:42:23,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742324_1500 (size=30955) 2024-11-19T08:42:23,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T08:42:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005803216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005803216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005803216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005803217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005803218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/af2a539d9c594f39be75e749b6a081da 2024-11-19T08:42:23,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/a2e7d54f3604496bbcf7d09f845e8671 is 50, key is test_row_0/B:col10/1732005742566/Put/seqid=0 2024-11-19T08:42:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742325_1501 (size=12001) 2024-11-19T08:42:23,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:23,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T08:42:23,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005803726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005803726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005803726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005803726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005803726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:23,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/a2e7d54f3604496bbcf7d09f845e8671 2024-11-19T08:42:23,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/5c2d19d5354740a18f46226539ae5f35 is 50, key is test_row_0/C:col10/1732005742566/Put/seqid=0 2024-11-19T08:42:23,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742326_1502 (size=12001) 2024-11-19T08:42:23,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:23,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:23,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:23,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:24,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:24,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:24,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:24,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:24,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:24,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:24,149 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T08:42:24,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:24,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:24,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:24,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:24,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:24,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:24,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/5c2d19d5354740a18f46226539ae5f35 2024-11-19T08:42:24,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/af2a539d9c594f39be75e749b6a081da as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/af2a539d9c594f39be75e749b6a081da 2024-11-19T08:42:24,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/af2a539d9c594f39be75e749b6a081da, entries=150, sequenceid=17, filesize=30.2 K 2024-11-19T08:42:24,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/a2e7d54f3604496bbcf7d09f845e8671 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a2e7d54f3604496bbcf7d09f845e8671 2024-11-19T08:42:24,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a2e7d54f3604496bbcf7d09f845e8671, entries=150, sequenceid=17, filesize=11.7 K 2024-11-19T08:42:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/5c2d19d5354740a18f46226539ae5f35 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5c2d19d5354740a18f46226539ae5f35 2024-11-19T08:42:24,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5c2d19d5354740a18f46226539ae5f35, entries=150, sequenceid=17, filesize=11.7 K 2024-11-19T08:42:24,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f0224c56755d95b541219e376c56cd39 in 1714ms, sequenceid=17, compaction requested=false 2024-11-19T08:42:24,281 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T08:42:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:24,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-19T08:42:24,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:24,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T08:42:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111968654042d91c4c92b2757110a1ef98c2_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005742587/Put/seqid=0 2024-11-19T08:42:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742327_1503 (size=12154) 2024-11-19T08:42:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T08:42:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:24,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005804746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005804746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005804747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,750 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005804750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005804750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:24,808 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111968654042d91c4c92b2757110a1ef98c2_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111968654042d91c4c92b2757110a1ef98c2_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/0e3edfd7a3ec46fe83de9c0c5930a430, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:24,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/0e3edfd7a3ec46fe83de9c0c5930a430 is 175, key is test_row_0/A:col10/1732005742587/Put/seqid=0 2024-11-19T08:42:24,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742328_1504 (size=30955) 2024-11-19T08:42:24,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005804856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005804856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005804866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005804866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005805065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005805065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005805076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005805076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,222 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/0e3edfd7a3ec46fe83de9c0c5930a430 2024-11-19T08:42:25,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/f37a136c56534424a82941475a24b5e9 is 50, key is test_row_0/B:col10/1732005742587/Put/seqid=0 2024-11-19T08:42:25,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742329_1505 (size=12001) 2024-11-19T08:42:25,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005805376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005805377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005805385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005805385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,632 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/f37a136c56534424a82941475a24b5e9 2024-11-19T08:42:25,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/c8253e54db1d447c8e1de9b947322b95 is 50, key is test_row_0/C:col10/1732005742587/Put/seqid=0 2024-11-19T08:42:25,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742330_1506 (size=12001) 2024-11-19T08:42:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005805885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005805885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005805896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005805896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:26,042 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/c8253e54db1d447c8e1de9b947322b95 2024-11-19T08:42:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/0e3edfd7a3ec46fe83de9c0c5930a430 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/0e3edfd7a3ec46fe83de9c0c5930a430 2024-11-19T08:42:26,049 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/0e3edfd7a3ec46fe83de9c0c5930a430, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T08:42:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/f37a136c56534424a82941475a24b5e9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/f37a136c56534424a82941475a24b5e9 2024-11-19T08:42:26,053 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/f37a136c56534424a82941475a24b5e9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T08:42:26,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/c8253e54db1d447c8e1de9b947322b95 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/c8253e54db1d447c8e1de9b947322b95 2024-11-19T08:42:26,058 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/c8253e54db1d447c8e1de9b947322b95, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T08:42:26,059 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for f0224c56755d95b541219e376c56cd39 in 1667ms, sequenceid=40, compaction requested=false 2024-11-19T08:42:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-19T08:42:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-19T08:42:26,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T08:42:26,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5020 sec 2024-11-19T08:42:26,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.5110 sec 2024-11-19T08:42:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T08:42:26,726 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:26,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T08:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T08:42:26,728 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:26,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:26,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:26,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:26,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T08:42:26,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b2570b6f455746e09e727b349c40ac7e_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005744748/Put/seqid=0 2024-11-19T08:42:26,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742331_1507 (size=14594) 2024-11-19T08:42:26,768 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:26,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b2570b6f455746e09e727b349c40ac7e_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b2570b6f455746e09e727b349c40ac7e_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:26,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/615df182c4bc4f8db14ee3c8b711b1d6, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:26,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/615df182c4bc4f8db14ee3c8b711b1d6 is 175, key is test_row_0/A:col10/1732005744748/Put/seqid=0 2024-11-19T08:42:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742332_1508 (size=39549) 2024-11-19T08:42:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005806829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T08:42:26,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:26,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005806906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:26,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005806906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005806906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:26,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005806906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:26,910 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:26,910 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:26,910 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:26,911 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_0' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:26,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005806936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:26,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:27,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:27,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:27,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T08:42:27,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005807146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:27,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/615df182c4bc4f8db14ee3c8b711b1d6 2024-11-19T08:42:27,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/e5d8a224b0bc4858a4c0cf7e509e94fe is 50, key is test_row_0/B:col10/1732005744748/Put/seqid=0 2024-11-19T08:42:27,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742333_1509 (size=12001) 2024-11-19T08:42:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:27,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:27,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T08:42:27,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005807456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:27,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:27,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:27,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:27,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/e5d8a224b0bc4858a4c0cf7e509e94fe 2024-11-19T08:42:27,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d2573f18c6a745bc8da009b5983367db is 50, key is test_row_0/C:col10/1732005744748/Put/seqid=0 2024-11-19T08:42:27,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742334_1510 (size=12001) 2024-11-19T08:42:27,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:27,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:27,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T08:42:27,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:27,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:27,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:27,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:27,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005807966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:27,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:27,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d2573f18c6a745bc8da009b5983367db 2024-11-19T08:42:27,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/615df182c4bc4f8db14ee3c8b711b1d6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/615df182c4bc4f8db14ee3c8b711b1d6 2024-11-19T08:42:28,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/615df182c4bc4f8db14ee3c8b711b1d6, entries=200, sequenceid=55, filesize=38.6 K 2024-11-19T08:42:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/e5d8a224b0bc4858a4c0cf7e509e94fe as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/e5d8a224b0bc4858a4c0cf7e509e94fe 2024-11-19T08:42:28,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/e5d8a224b0bc4858a4c0cf7e509e94fe, entries=150, sequenceid=55, filesize=11.7 K 2024-11-19T08:42:28,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d2573f18c6a745bc8da009b5983367db as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d2573f18c6a745bc8da009b5983367db 2024-11-19T08:42:28,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d2573f18c6a745bc8da009b5983367db, entries=150, sequenceid=55, filesize=11.7 K 2024-11-19T08:42:28,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f0224c56755d95b541219e376c56cd39 in 1249ms, sequenceid=55, compaction requested=true 2024-11-19T08:42:28,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:28,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:28,009 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:28,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:28,009 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:28,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/A is initiating minor compaction (all files) 2024-11-19T08:42:28,009 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/B is initiating minor compaction (all files) 2024-11-19T08:42:28,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/A in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:28,009 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/B in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:28,009 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a2e7d54f3604496bbcf7d09f845e8671, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/f37a136c56534424a82941475a24b5e9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/e5d8a224b0bc4858a4c0cf7e509e94fe] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.2 K 2024-11-19T08:42:28,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/af2a539d9c594f39be75e749b6a081da, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/0e3edfd7a3ec46fe83de9c0c5930a430, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/615df182c4bc4f8db14ee3c8b711b1d6] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=99.1 K 2024-11-19T08:42:28,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:28,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/af2a539d9c594f39be75e749b6a081da, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/0e3edfd7a3ec46fe83de9c0c5930a430, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/615df182c4bc4f8db14ee3c8b711b1d6] 2024-11-19T08:42:28,010 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e7d54f3604496bbcf7d09f845e8671, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732005742566 2024-11-19T08:42:28,010 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting af2a539d9c594f39be75e749b6a081da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732005742566 2024-11-19T08:42:28,010 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f37a136c56534424a82941475a24b5e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732005742574 2024-11-19T08:42:28,010 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e3edfd7a3ec46fe83de9c0c5930a430, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732005742574 2024-11-19T08:42:28,010 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d8a224b0bc4858a4c0cf7e509e94fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732005744743 2024-11-19T08:42:28,010 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 615df182c4bc4f8db14ee3c8b711b1d6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732005744743 2024-11-19T08:42:28,014 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:28,015 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:28,016 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/b75f73f533b3401ab3e82b38a814db6c is 50, key is test_row_0/B:col10/1732005744748/Put/seqid=0 2024-11-19T08:42:28,018 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411196301224c6c8049a2bb974f6e56856d93_f0224c56755d95b541219e376c56cd39 store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:28,019 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411196301224c6c8049a2bb974f6e56856d93_f0224c56755d95b541219e376c56cd39, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:28,019 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196301224c6c8049a2bb974f6e56856d93_f0224c56755d95b541219e376c56cd39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:28,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742335_1511 (size=12104) 2024-11-19T08:42:28,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742336_1512 (size=4469) 2024-11-19T08:42:28,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T08:42:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:28,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T08:42:28,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e49a1040f2004738a9af0fd649d5bc3e_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005746828/Put/seqid=0 2024-11-19T08:42:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742337_1513 (size=12154) 2024-11-19T08:42:28,434 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/b75f73f533b3401ab3e82b38a814db6c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b75f73f533b3401ab3e82b38a814db6c 2024-11-19T08:42:28,451 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#A#compaction#436 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:28,452 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/bd1e358adc064d5a8f5027ce640d2617 is 175, key is test_row_0/A:col10/1732005744748/Put/seqid=0 2024-11-19T08:42:28,458 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/B of f0224c56755d95b541219e376c56cd39 into b75f73f533b3401ab3e82b38a814db6c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:28,458 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:28,458 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/B, priority=13, startTime=1732005748009; duration=0sec 2024-11-19T08:42:28,458 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:28,458 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:B 2024-11-19T08:42:28,458 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:28,459 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:28,459 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/C is initiating minor compaction (all files) 2024-11-19T08:42:28,459 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/C in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:28,459 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5c2d19d5354740a18f46226539ae5f35, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/c8253e54db1d447c8e1de9b947322b95, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d2573f18c6a745bc8da009b5983367db] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.2 K 2024-11-19T08:42:28,460 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2d19d5354740a18f46226539ae5f35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732005742566 2024-11-19T08:42:28,460 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting c8253e54db1d447c8e1de9b947322b95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732005742574 2024-11-19T08:42:28,460 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d2573f18c6a745bc8da009b5983367db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732005744743 2024-11-19T08:42:28,482 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:28,482 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/08efffeba9f94eb796fc33443f3e38ec is 50, key is test_row_0/C:col10/1732005744748/Put/seqid=0 2024-11-19T08:42:28,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742338_1514 (size=31058) 2024-11-19T08:42:28,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742339_1515 (size=12104) 2024-11-19T08:42:28,498 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/08efffeba9f94eb796fc33443f3e38ec as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08efffeba9f94eb796fc33443f3e38ec 2024-11-19T08:42:28,503 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/C of f0224c56755d95b541219e376c56cd39 into 08efffeba9f94eb796fc33443f3e38ec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:28,503 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:28,503 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/C, priority=13, startTime=1732005748009; duration=0sec 2024-11-19T08:42:28,503 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:28,503 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:C 2024-11-19T08:42:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:28,519 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e49a1040f2004738a9af0fd649d5bc3e_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e49a1040f2004738a9af0fd649d5bc3e_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:28,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/3defcc68058f493fbbecfe586c25539d, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:28,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/3defcc68058f493fbbecfe586c25539d is 175, key is test_row_0/A:col10/1732005746828/Put/seqid=0 2024-11-19T08:42:28,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742340_1516 (size=30955) 2024-11-19T08:42:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T08:42:28,900 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/bd1e358adc064d5a8f5027ce640d2617 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/bd1e358adc064d5a8f5027ce640d2617 2024-11-19T08:42:28,907 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/A of f0224c56755d95b541219e376c56cd39 into bd1e358adc064d5a8f5027ce640d2617(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:28,907 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:28,907 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/A, priority=13, startTime=1732005748008; duration=0sec 2024-11-19T08:42:28,907 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:28,907 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:A 2024-11-19T08:42:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:28,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:28,928 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/3defcc68058f493fbbecfe586c25539d 2024-11-19T08:42:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/cca72c9f31f648ff9ccf24d6747c8c52 is 50, key is test_row_0/B:col10/1732005746828/Put/seqid=0 2024-11-19T08:42:28,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742341_1517 (size=12001) 2024-11-19T08:42:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005808942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:28,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005808943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005808947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005808949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:28,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005808976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:28,981 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005809056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005809056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005809057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005809057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005809266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005809266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005809267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005809267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/cca72c9f31f648ff9ccf24d6747c8c52 2024-11-19T08:42:29,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/587298349b8742b49833baf3180a362f is 50, key is test_row_0/C:col10/1732005746828/Put/seqid=0 2024-11-19T08:42:29,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742342_1518 (size=12001) 2024-11-19T08:42:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005809575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005809576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005809576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005809576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:29,757 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/587298349b8742b49833baf3180a362f 2024-11-19T08:42:29,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/3defcc68058f493fbbecfe586c25539d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3defcc68058f493fbbecfe586c25539d 2024-11-19T08:42:29,764 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3defcc68058f493fbbecfe586c25539d, entries=150, sequenceid=76, filesize=30.2 K 2024-11-19T08:42:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/cca72c9f31f648ff9ccf24d6747c8c52 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/cca72c9f31f648ff9ccf24d6747c8c52 2024-11-19T08:42:29,769 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/cca72c9f31f648ff9ccf24d6747c8c52, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T08:42:29,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/587298349b8742b49833baf3180a362f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587298349b8742b49833baf3180a362f 2024-11-19T08:42:29,776 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587298349b8742b49833baf3180a362f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T08:42:29,777 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for f0224c56755d95b541219e376c56cd39 in 1674ms, sequenceid=76, compaction requested=false 2024-11-19T08:42:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T08:42:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T08:42:29,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T08:42:29,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0500 sec 2024-11-19T08:42:29,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.0570 sec 2024-11-19T08:42:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:30,088 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:30,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T08:42:30,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111920e2454018db4ae68495450239240af3_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005750087/Put/seqid=0 2024-11-19T08:42:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742343_1519 (size=14594) 2024-11-19T08:42:30,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005810108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005810109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005810109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005810110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005810216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005810216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005810216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005810226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005810426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005810427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005810427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005810436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:30,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111920e2454018db4ae68495450239240af3_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111920e2454018db4ae68495450239240af3_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:30,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d2559b11f0704a00a7c514a36e218b10, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:30,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d2559b11f0704a00a7c514a36e218b10 is 175, key is test_row_0/A:col10/1732005750087/Put/seqid=0 2024-11-19T08:42:30,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742344_1520 (size=39549) 2024-11-19T08:42:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005810736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005810736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005810736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005810745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T08:42:30,896 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:30,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-19T08:42:30,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T08:42:30,899 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:30,900 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:30,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:30,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d2559b11f0704a00a7c514a36e218b10 2024-11-19T08:42:30,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/6adf7e8ac0ac4005b814f67d160c9b05 is 50, key is test_row_0/B:col10/1732005750087/Put/seqid=0 2024-11-19T08:42:30,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742345_1521 (size=12001) 2024-11-19T08:42:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T08:42:31,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005811006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:31,008 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:31,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T08:42:31,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:31,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T08:42:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T08:42:31,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005811245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:31,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005811245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005811245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:31,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005811256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:31,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/6adf7e8ac0ac4005b814f67d160c9b05 2024-11-19T08:42:31,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0cfcfac7ca8549c7895b04d87107d703 is 50, key is test_row_0/C:col10/1732005750087/Put/seqid=0 2024-11-19T08:42:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742346_1522 (size=12001) 2024-11-19T08:42:31,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T08:42:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:31,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T08:42:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T08:42:31,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T08:42:31,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:31,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:31,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0cfcfac7ca8549c7895b04d87107d703 2024-11-19T08:42:31,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d2559b11f0704a00a7c514a36e218b10 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d2559b11f0704a00a7c514a36e218b10 2024-11-19T08:42:31,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d2559b11f0704a00a7c514a36e218b10, entries=200, sequenceid=95, filesize=38.6 K 2024-11-19T08:42:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/6adf7e8ac0ac4005b814f67d160c9b05 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6adf7e8ac0ac4005b814f67d160c9b05 2024-11-19T08:42:31,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6adf7e8ac0ac4005b814f67d160c9b05, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T08:42:31,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0cfcfac7ca8549c7895b04d87107d703 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cfcfac7ca8549c7895b04d87107d703 2024-11-19T08:42:31,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cfcfac7ca8549c7895b04d87107d703, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T08:42:31,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f0224c56755d95b541219e376c56cd39 in 1689ms, sequenceid=95, compaction requested=true 2024-11-19T08:42:31,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:31,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:31,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:31,778 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:31,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:31,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:31,778 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:31,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:31,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:31,778 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:31,778 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/B is initiating minor compaction (all files) 2024-11-19T08:42:31,779 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/B in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,779 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b75f73f533b3401ab3e82b38a814db6c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/cca72c9f31f648ff9ccf24d6747c8c52, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6adf7e8ac0ac4005b814f67d160c9b05] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.3 K 2024-11-19T08:42:31,779 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:31,779 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/A is initiating minor compaction (all files) 2024-11-19T08:42:31,779 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/A in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,779 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/bd1e358adc064d5a8f5027ce640d2617, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3defcc68058f493fbbecfe586c25539d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d2559b11f0704a00a7c514a36e218b10] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=99.2 K 2024-11-19T08:42:31,779 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,779 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/bd1e358adc064d5a8f5027ce640d2617, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3defcc68058f493fbbecfe586c25539d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d2559b11f0704a00a7c514a36e218b10] 2024-11-19T08:42:31,779 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b75f73f533b3401ab3e82b38a814db6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732005744743 2024-11-19T08:42:31,780 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting cca72c9f31f648ff9ccf24d6747c8c52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005746815 2024-11-19T08:42:31,780 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd1e358adc064d5a8f5027ce640d2617, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732005744743 2024-11-19T08:42:31,781 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6adf7e8ac0ac4005b814f67d160c9b05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005748946 2024-11-19T08:42:31,781 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3defcc68058f493fbbecfe586c25539d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005746815 2024-11-19T08:42:31,782 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2559b11f0704a00a7c514a36e218b10, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005748940 2024-11-19T08:42:31,792 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:31,792 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/d0d361f9ed614f52b5676ce312618a5a is 50, key is test_row_0/B:col10/1732005750087/Put/seqid=0 2024-11-19T08:42:31,794 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:31,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742347_1523 (size=12207) 2024-11-19T08:42:31,803 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111923798ec6d3b54eb2bc1ee92a8ea59076_f0224c56755d95b541219e376c56cd39 store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:31,805 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111923798ec6d3b54eb2bc1ee92a8ea59076_f0224c56755d95b541219e376c56cd39, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:31,805 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111923798ec6d3b54eb2bc1ee92a8ea59076_f0224c56755d95b541219e376c56cd39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:31,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742348_1524 (size=4469) 2024-11-19T08:42:31,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T08:42:31,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:31,816 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T08:42:31,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c4a02cdd86814456b14fab7de5cb68f8_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005750107/Put/seqid=0 2024-11-19T08:42:31,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742349_1525 (size=12154) 2024-11-19T08:42:31,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:31,837 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c4a02cdd86814456b14fab7de5cb68f8_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c4a02cdd86814456b14fab7de5cb68f8_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ef2bf1fc612e46a99cdd61d54c7d85e5, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:31,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ef2bf1fc612e46a99cdd61d54c7d85e5 is 175, key is test_row_0/A:col10/1732005750107/Put/seqid=0 2024-11-19T08:42:31,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742350_1526 (size=30955) 2024-11-19T08:42:31,841 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ef2bf1fc612e46a99cdd61d54c7d85e5 2024-11-19T08:42:31,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/a9e1193dcef140fb905c986ca635c85b is 50, key is test_row_0/B:col10/1732005750107/Put/seqid=0 2024-11-19T08:42:31,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742351_1527 (size=12001) 2024-11-19T08:42:31,850 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/a9e1193dcef140fb905c986ca635c85b 2024-11-19T08:42:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/82dfa7e6449d41f7b0849e5b9f92011e is 50, key is test_row_0/C:col10/1732005750107/Put/seqid=0 2024-11-19T08:42:31,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742352_1528 (size=12001) 2024-11-19T08:42:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T08:42:32,208 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/d0d361f9ed614f52b5676ce312618a5a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d0d361f9ed614f52b5676ce312618a5a 2024-11-19T08:42:32,211 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#A#compaction#445 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:32,212 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/e74087b6d5194dbb99d84f0d49c500dc is 175, key is test_row_0/A:col10/1732005750087/Put/seqid=0 2024-11-19T08:42:32,213 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/B of f0224c56755d95b541219e376c56cd39 into d0d361f9ed614f52b5676ce312618a5a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:32,213 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:32,213 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/B, priority=13, startTime=1732005751778; duration=0sec 2024-11-19T08:42:32,213 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:32,213 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:B 2024-11-19T08:42:32,213 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:32,214 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:32,214 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/C is initiating minor compaction (all files) 2024-11-19T08:42:32,214 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/C in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:32,214 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08efffeba9f94eb796fc33443f3e38ec, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587298349b8742b49833baf3180a362f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cfcfac7ca8549c7895b04d87107d703] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.3 K 2024-11-19T08:42:32,214 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 08efffeba9f94eb796fc33443f3e38ec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732005744743 2024-11-19T08:42:32,215 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 587298349b8742b49833baf3180a362f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732005746815 2024-11-19T08:42:32,215 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cfcfac7ca8549c7895b04d87107d703, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005748946 2024-11-19T08:42:32,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742353_1529 (size=31161) 2024-11-19T08:42:32,224 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:32,225 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/6cc49563f8934197acdf1160000b3179 is 50, key is test_row_0/C:col10/1732005750087/Put/seqid=0 2024-11-19T08:42:32,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742354_1530 (size=12207) 2024-11-19T08:42:32,241 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/6cc49563f8934197acdf1160000b3179 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6cc49563f8934197acdf1160000b3179 2024-11-19T08:42:32,248 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/C of f0224c56755d95b541219e376c56cd39 into 6cc49563f8934197acdf1160000b3179(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:32,249 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:32,249 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/C, priority=13, startTime=1732005751778; duration=0sec 2024-11-19T08:42:32,249 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:32,249 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:C 2024-11-19T08:42:32,259 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/82dfa7e6449d41f7b0849e5b9f92011e 2024-11-19T08:42:32,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ef2bf1fc612e46a99cdd61d54c7d85e5 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ef2bf1fc612e46a99cdd61d54c7d85e5 2024-11-19T08:42:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:32,271 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ef2bf1fc612e46a99cdd61d54c7d85e5, entries=150, sequenceid=115, filesize=30.2 K 2024-11-19T08:42:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/a9e1193dcef140fb905c986ca635c85b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a9e1193dcef140fb905c986ca635c85b 2024-11-19T08:42:32,276 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a9e1193dcef140fb905c986ca635c85b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T08:42:32,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/82dfa7e6449d41f7b0849e5b9f92011e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/82dfa7e6449d41f7b0849e5b9f92011e 2024-11-19T08:42:32,280 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/82dfa7e6449d41f7b0849e5b9f92011e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T08:42:32,281 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=67.09 KB/68700 for f0224c56755d95b541219e376c56cd39 in 465ms, sequenceid=115, compaction requested=false 2024-11-19T08:42:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:32,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-19T08:42:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-19T08:42:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:32,285 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:32,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T08:42:32,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T08:42:32,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-11-19T08:42:32,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.3890 sec 2024-11-19T08:42:32,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111938cfaf31f13e43e6b7c1e6d97ddc4504_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005752284/Put/seqid=0 2024-11-19T08:42:32,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742355_1531 (size=12254) 2024-11-19T08:42:32,296 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:32,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111938cfaf31f13e43e6b7c1e6d97ddc4504_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111938cfaf31f13e43e6b7c1e6d97ddc4504_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:32,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/b5d85d69bcfe49158442262717c2aa80, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:32,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/b5d85d69bcfe49158442262717c2aa80 is 175, key is test_row_0/A:col10/1732005752284/Put/seqid=0 2024-11-19T08:42:32,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742356_1532 (size=31055) 2024-11-19T08:42:32,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005812311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005812312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005812312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005812312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005812426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005812426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005812426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005812426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,620 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/e74087b6d5194dbb99d84f0d49c500dc as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e74087b6d5194dbb99d84f0d49c500dc 2024-11-19T08:42:32,623 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/A of f0224c56755d95b541219e376c56cd39 into e74087b6d5194dbb99d84f0d49c500dc(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:32,624 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:32,624 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/A, priority=13, startTime=1732005751777; duration=0sec 2024-11-19T08:42:32,624 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:32,624 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:A 2024-11-19T08:42:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005812636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005812636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005812636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005812636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,714 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/b5d85d69bcfe49158442262717c2aa80 2024-11-19T08:42:32,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/d2028bea642946dc9fcdd3f25886d29f is 50, key is test_row_0/B:col10/1732005752284/Put/seqid=0 2024-11-19T08:42:32,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742357_1533 (size=12101) 2024-11-19T08:42:32,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005812946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005812946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005812946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005812946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T08:42:33,046 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:33,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-19T08:42:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T08:42:33,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:33,049 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:33,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:33,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/d2028bea642946dc9fcdd3f25886d29f 2024-11-19T08:42:33,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/587415305c744e26a4c31c57fc7f9fac is 50, key is test_row_0/C:col10/1732005752284/Put/seqid=0 2024-11-19T08:42:33,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742358_1534 (size=12101) 2024-11-19T08:42:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T08:42:33,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T08:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T08:42:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T08:42:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005813456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005813456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005813456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005813456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:33,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T08:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:33,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/587415305c744e26a4c31c57fc7f9fac 2024-11-19T08:42:33,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/b5d85d69bcfe49158442262717c2aa80 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b5d85d69bcfe49158442262717c2aa80 2024-11-19T08:42:33,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b5d85d69bcfe49158442262717c2aa80, entries=150, sequenceid=132, filesize=30.3 K 2024-11-19T08:42:33,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/d2028bea642946dc9fcdd3f25886d29f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d2028bea642946dc9fcdd3f25886d29f 2024-11-19T08:42:33,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d2028bea642946dc9fcdd3f25886d29f, entries=150, sequenceid=132, filesize=11.8 K 2024-11-19T08:42:33,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/587415305c744e26a4c31c57fc7f9fac as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587415305c744e26a4c31c57fc7f9fac 2024-11-19T08:42:33,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587415305c744e26a4c31c57fc7f9fac, entries=150, sequenceid=132, filesize=11.8 K 2024-11-19T08:42:33,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f0224c56755d95b541219e376c56cd39 in 1260ms, sequenceid=132, compaction requested=true 2024-11-19T08:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:33,545 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:33,545 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:33,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:33,547 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:33,547 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:33,547 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/A is initiating minor compaction (all files) 2024-11-19T08:42:33,547 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/B is initiating minor compaction (all files) 2024-11-19T08:42:33,547 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/A in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,547 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/B in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,547 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e74087b6d5194dbb99d84f0d49c500dc, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ef2bf1fc612e46a99cdd61d54c7d85e5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b5d85d69bcfe49158442262717c2aa80] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=91.0 K 2024-11-19T08:42:33,547 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d0d361f9ed614f52b5676ce312618a5a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a9e1193dcef140fb905c986ca635c85b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d2028bea642946dc9fcdd3f25886d29f] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.5 K 2024-11-19T08:42:33,547 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,547 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e74087b6d5194dbb99d84f0d49c500dc, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ef2bf1fc612e46a99cdd61d54c7d85e5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b5d85d69bcfe49158442262717c2aa80] 2024-11-19T08:42:33,548 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d361f9ed614f52b5676ce312618a5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005748946 2024-11-19T08:42:33,548 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74087b6d5194dbb99d84f0d49c500dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005748946 2024-11-19T08:42:33,548 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e1193dcef140fb905c986ca635c85b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732005750107 2024-11-19T08:42:33,548 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2bf1fc612e46a99cdd61d54c7d85e5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732005750107 2024-11-19T08:42:33,548 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d2028bea642946dc9fcdd3f25886d29f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732005752277 2024-11-19T08:42:33,548 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d85d69bcfe49158442262717c2aa80, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732005752277 2024-11-19T08:42:33,552 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:33,563 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e020a9a20f7245aba5c60049474ab31c_f0224c56755d95b541219e376c56cd39 store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:33,564 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:33,564 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/5cdf8fa4d07149cc99a99b04bafe6cad is 50, key is test_row_0/B:col10/1732005752284/Put/seqid=0 2024-11-19T08:42:33,565 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e020a9a20f7245aba5c60049474ab31c_f0224c56755d95b541219e376c56cd39, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:33,565 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e020a9a20f7245aba5c60049474ab31c_f0224c56755d95b541219e376c56cd39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:33,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742360_1536 (size=4469) 2024-11-19T08:42:33,572 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#A#compaction#453 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:33,573 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/3cc71816cd274087b906e9540d3001d8 is 175, key is test_row_0/A:col10/1732005752284/Put/seqid=0 2024-11-19T08:42:33,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742359_1535 (size=12409) 2024-11-19T08:42:33,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742361_1537 (size=31363) 2024-11-19T08:42:33,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T08:42:33,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:33,658 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T08:42:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195f02a11471624dc9b7ab0c608124e6b8_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005752311/Put/seqid=0 2024-11-19T08:42:33,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742362_1538 (size=12304) 2024-11-19T08:42:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T08:42:34,000 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/5cdf8fa4d07149cc99a99b04bafe6cad as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/5cdf8fa4d07149cc99a99b04bafe6cad 2024-11-19T08:42:34,003 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/3cc71816cd274087b906e9540d3001d8 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3cc71816cd274087b906e9540d3001d8 2024-11-19T08:42:34,007 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/A of f0224c56755d95b541219e376c56cd39 into 3cc71816cd274087b906e9540d3001d8(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:34,007 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:34,008 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/A, priority=13, startTime=1732005753545; duration=0sec 2024-11-19T08:42:34,008 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:34,008 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:A 2024-11-19T08:42:34,008 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:34,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:34,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/C is initiating minor compaction (all files) 2024-11-19T08:42:34,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/C in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:34,009 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6cc49563f8934197acdf1160000b3179, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/82dfa7e6449d41f7b0849e5b9f92011e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587415305c744e26a4c31c57fc7f9fac] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.5 K 2024-11-19T08:42:34,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cc49563f8934197acdf1160000b3179, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732005748946 2024-11-19T08:42:34,009 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82dfa7e6449d41f7b0849e5b9f92011e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732005750107 2024-11-19T08:42:34,010 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 587415305c744e26a4c31c57fc7f9fac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732005752277 2024-11-19T08:42:34,015 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:34,016 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d3c941158d7b4667a11730203d1d6b03 is 50, key is test_row_0/C:col10/1732005752284/Put/seqid=0 2024-11-19T08:42:34,019 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/B of f0224c56755d95b541219e376c56cd39 into 5cdf8fa4d07149cc99a99b04bafe6cad(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:34,019 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:34,019 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/B, priority=13, startTime=1732005753545; duration=0sec 2024-11-19T08:42:34,019 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:34,019 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:B 2024-11-19T08:42:34,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742363_1539 (size=12409) 2024-11-19T08:42:34,023 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d3c941158d7b4667a11730203d1d6b03 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d3c941158d7b4667a11730203d1d6b03 2024-11-19T08:42:34,027 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/C of f0224c56755d95b541219e376c56cd39 into d3c941158d7b4667a11730203d1d6b03(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:34,027 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:34,027 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/C, priority=13, startTime=1732005753545; duration=0sec 2024-11-19T08:42:34,028 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:34,028 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:C 2024-11-19T08:42:34,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:34,071 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195f02a11471624dc9b7ab0c608124e6b8_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195f02a11471624dc9b7ab0c608124e6b8_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:34,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d192a8d73b894e52bfb771c6a837ea3d, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:34,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d192a8d73b894e52bfb771c6a837ea3d is 175, key is test_row_0/A:col10/1732005752311/Put/seqid=0 2024-11-19T08:42:34,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742364_1540 (size=31105) 2024-11-19T08:42:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T08:42:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:34,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:34,476 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d192a8d73b894e52bfb771c6a837ea3d 2024-11-19T08:42:34,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/2f2b7b3ad6024b738baacd2f09de68cb is 50, key is test_row_0/B:col10/1732005752311/Put/seqid=0 2024-11-19T08:42:34,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005814478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742365_1541 (size=12151) 2024-11-19T08:42:34,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005814481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005814481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005814483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005814586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005814596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005814596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005814596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005814796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005814806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005814806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005814807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:34,885 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/2f2b7b3ad6024b738baacd2f09de68cb 2024-11-19T08:42:34,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/08537430a25647f38cabe089a1f12ce2 is 50, key is test_row_0/C:col10/1732005752311/Put/seqid=0 2024-11-19T08:42:34,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742366_1542 (size=12151) 2024-11-19T08:42:34,895 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/08537430a25647f38cabe089a1f12ce2 2024-11-19T08:42:34,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/d192a8d73b894e52bfb771c6a837ea3d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d192a8d73b894e52bfb771c6a837ea3d 2024-11-19T08:42:34,901 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d192a8d73b894e52bfb771c6a837ea3d, entries=150, sequenceid=154, filesize=30.4 K 2024-11-19T08:42:34,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/2f2b7b3ad6024b738baacd2f09de68cb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/2f2b7b3ad6024b738baacd2f09de68cb 2024-11-19T08:42:34,905 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/2f2b7b3ad6024b738baacd2f09de68cb, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T08:42:34,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/08537430a25647f38cabe089a1f12ce2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08537430a25647f38cabe089a1f12ce2 2024-11-19T08:42:34,925 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08537430a25647f38cabe089a1f12ce2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T08:42:34,926 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for f0224c56755d95b541219e376c56cd39 in 1268ms, sequenceid=154, compaction requested=false 2024-11-19T08:42:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:34,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-19T08:42:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-19T08:42:34,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T08:42:34,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-19T08:42:34,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8830 sec 2024-11-19T08:42:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:35,028 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:35,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T08:42:35,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119583de29e8eca497da18796bec58fd8ce_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005754480/Put/seqid=0 2024-11-19T08:42:35,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742367_1543 (size=14794) 2024-11-19T08:42:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005815099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005815106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005815115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005815115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005815126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T08:42:35,197 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:35,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-19T08:42:35,199 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:35,199 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:35,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T08:42:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005815206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T08:42:35,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:35,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:35,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005815415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,439 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:35,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119583de29e8eca497da18796bec58fd8ce_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119583de29e8eca497da18796bec58fd8ce_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:35,442 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/787a2509e39b4d999a6cce049ccd7716, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:35,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/787a2509e39b4d999a6cce049ccd7716 is 175, key is test_row_0/A:col10/1732005754480/Put/seqid=0 2024-11-19T08:42:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742368_1544 (size=39749) 2024-11-19T08:42:35,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T08:42:35,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005815616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005815626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005815626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005815635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:35,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005815726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:35,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:35,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:35,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T08:42:35,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/787a2509e39b4d999a6cce049ccd7716 2024-11-19T08:42:35,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/ace99d3f297b46e7b472f22fe99559f1 is 50, key is test_row_0/B:col10/1732005754480/Put/seqid=0 2024-11-19T08:42:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742369_1545 (size=12151) 2024-11-19T08:42:35,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:35,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:35,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:35,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:36,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:36,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:36,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:36,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:36,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005816236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/ace99d3f297b46e7b472f22fe99559f1 2024-11-19T08:42:36,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/f7703581d3a2463a9bd37657cc06c584 is 50, key is test_row_0/C:col10/1732005754480/Put/seqid=0 2024-11-19T08:42:36,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742370_1546 (size=12151) 2024-11-19T08:42:36,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/f7703581d3a2463a9bd37657cc06c584 2024-11-19T08:42:36,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/787a2509e39b4d999a6cce049ccd7716 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/787a2509e39b4d999a6cce049ccd7716 2024-11-19T08:42:36,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:36,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/787a2509e39b4d999a6cce049ccd7716, entries=200, sequenceid=173, filesize=38.8 K 2024-11-19T08:42:36,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/ace99d3f297b46e7b472f22fe99559f1 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/ace99d3f297b46e7b472f22fe99559f1 2024-11-19T08:42:36,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/ace99d3f297b46e7b472f22fe99559f1, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T08:42:36,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/f7703581d3a2463a9bd37657cc06c584 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f7703581d3a2463a9bd37657cc06c584 2024-11-19T08:42:36,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f7703581d3a2463a9bd37657cc06c584, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T08:42:36,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f0224c56755d95b541219e376c56cd39 in 1248ms, sequenceid=173, compaction requested=true 2024-11-19T08:42:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:36,276 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:36,276 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/A is initiating minor compaction (all files) 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/B is initiating minor compaction (all files) 2024-11-19T08:42:36,277 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/A in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,277 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/B in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,277 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3cc71816cd274087b906e9540d3001d8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d192a8d73b894e52bfb771c6a837ea3d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/787a2509e39b4d999a6cce049ccd7716] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=99.8 K 2024-11-19T08:42:36,277 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/5cdf8fa4d07149cc99a99b04bafe6cad, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/2f2b7b3ad6024b738baacd2f09de68cb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/ace99d3f297b46e7b472f22fe99559f1] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.9 K 2024-11-19T08:42:36,277 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3cc71816cd274087b906e9540d3001d8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d192a8d73b894e52bfb771c6a837ea3d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/787a2509e39b4d999a6cce049ccd7716] 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cdf8fa4d07149cc99a99b04bafe6cad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732005752277 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc71816cd274087b906e9540d3001d8, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732005752277 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2b7b3ad6024b738baacd2f09de68cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732005752305 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting d192a8d73b894e52bfb771c6a837ea3d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732005752305 2024-11-19T08:42:36,277 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting ace99d3f297b46e7b472f22fe99559f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732005754473 2024-11-19T08:42:36,278 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787a2509e39b4d999a6cce049ccd7716, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732005754473 2024-11-19T08:42:36,281 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:36,283 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#B#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:36,283 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199729b656ce3b4f10a5dec28bbdbd747c_f0224c56755d95b541219e376c56cd39 store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:36,283 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/aaf1c7d37c9c4144af8a22ffd5cc3ad9 is 50, key is test_row_0/B:col10/1732005754480/Put/seqid=0 2024-11-19T08:42:36,285 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199729b656ce3b4f10a5dec28bbdbd747c_f0224c56755d95b541219e376c56cd39, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:36,285 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199729b656ce3b4f10a5dec28bbdbd747c_f0224c56755d95b541219e376c56cd39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:36,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742371_1547 (size=12561) 2024-11-19T08:42:36,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742372_1548 (size=4469) 2024-11-19T08:42:36,300 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#A#compaction#463 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:36,301 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/52a229f07bfa45278e97917d6c9b715a is 175, key is test_row_0/A:col10/1732005754480/Put/seqid=0 2024-11-19T08:42:36,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742373_1549 (size=31515) 2024-11-19T08:42:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T08:42:36,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T08:42:36,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,421 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T08:42:36,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119326035270db94a1c89093247ae3990bd_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005755098/Put/seqid=0 2024-11-19T08:42:36,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742374_1550 (size=12304) 2024-11-19T08:42:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:36,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:36,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005816655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,659 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005816656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005816657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005816659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,690 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/aaf1c7d37c9c4144af8a22ffd5cc3ad9 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/aaf1c7d37c9c4144af8a22ffd5cc3ad9 2024-11-19T08:42:36,693 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/B of f0224c56755d95b541219e376c56cd39 into aaf1c7d37c9c4144af8a22ffd5cc3ad9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:36,693 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:36,693 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/B, priority=13, startTime=1732005756276; duration=0sec 2024-11-19T08:42:36,693 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:36,693 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:B 2024-11-19T08:42:36,693 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:36,694 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:36,694 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/C is initiating minor compaction (all files) 2024-11-19T08:42:36,694 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/C in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,694 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d3c941158d7b4667a11730203d1d6b03, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08537430a25647f38cabe089a1f12ce2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f7703581d3a2463a9bd37657cc06c584] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=35.9 K 2024-11-19T08:42:36,694 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c941158d7b4667a11730203d1d6b03, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732005752277 2024-11-19T08:42:36,694 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 08537430a25647f38cabe089a1f12ce2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732005752305 2024-11-19T08:42:36,695 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f7703581d3a2463a9bd37657cc06c584, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732005754473 2024-11-19T08:42:36,700 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:36,701 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0f7b197ee4b3447ab59f1fffe84c7520 is 50, key is test_row_0/C:col10/1732005754480/Put/seqid=0 2024-11-19T08:42:36,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742375_1551 (size=12561) 2024-11-19T08:42:36,707 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/52a229f07bfa45278e97917d6c9b715a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/52a229f07bfa45278e97917d6c9b715a 2024-11-19T08:42:36,710 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/A of f0224c56755d95b541219e376c56cd39 into 52a229f07bfa45278e97917d6c9b715a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:36,710 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:36,710 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/A, priority=13, startTime=1732005756276; duration=0sec 2024-11-19T08:42:36,710 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:36,710 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:A 2024-11-19T08:42:36,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005816765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005816765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005816776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:36,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:36,836 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119326035270db94a1c89093247ae3990bd_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119326035270db94a1c89093247ae3990bd_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/510bcb2cd8844301ace60e1cbbfa5e48, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:36,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/510bcb2cd8844301ace60e1cbbfa5e48 is 175, key is test_row_0/A:col10/1732005755098/Put/seqid=0 2024-11-19T08:42:36,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742376_1552 (size=31105) 2024-11-19T08:42:36,840 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/510bcb2cd8844301ace60e1cbbfa5e48 2024-11-19T08:42:36,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/645ef749cc1247a0892129e5c80b4f23 is 50, key is test_row_0/B:col10/1732005755098/Put/seqid=0 2024-11-19T08:42:36,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742377_1553 (size=12151) 2024-11-19T08:42:36,848 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/645ef749cc1247a0892129e5c80b4f23 2024-11-19T08:42:36,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/df3979c8bb574a7c81461442e0035471 is 50, key is test_row_0/C:col10/1732005755098/Put/seqid=0 2024-11-19T08:42:36,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742378_1554 (size=12151) 2024-11-19T08:42:36,857 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/df3979c8bb574a7c81461442e0035471 2024-11-19T08:42:36,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/510bcb2cd8844301ace60e1cbbfa5e48 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/510bcb2cd8844301ace60e1cbbfa5e48 2024-11-19T08:42:36,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/510bcb2cd8844301ace60e1cbbfa5e48, entries=150, sequenceid=193, filesize=30.4 K 2024-11-19T08:42:36,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/645ef749cc1247a0892129e5c80b4f23 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/645ef749cc1247a0892129e5c80b4f23 2024-11-19T08:42:36,865 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/645ef749cc1247a0892129e5c80b4f23, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T08:42:36,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/df3979c8bb574a7c81461442e0035471 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/df3979c8bb574a7c81461442e0035471 2024-11-19T08:42:36,869 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/df3979c8bb574a7c81461442e0035471, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T08:42:36,869 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f0224c56755d95b541219e376c56cd39 in 449ms, sequenceid=193, compaction requested=false 2024-11-19T08:42:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-19T08:42:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-19T08:42:36,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-19T08:42:36,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6720 sec 2024-11-19T08:42:36,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6760 sec 2024-11-19T08:42:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:36,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:36,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T08:42:36,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111937b177a3d6004f7da21fb9d98990a751_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005756977/Put/seqid=0 2024-11-19T08:42:36,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742379_1555 (size=14794) 2024-11-19T08:42:37,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005817005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005817006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005817009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,107 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0f7b197ee4b3447ab59f1fffe84c7520 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0f7b197ee4b3447ab59f1fffe84c7520 2024-11-19T08:42:37,111 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/C of f0224c56755d95b541219e376c56cd39 into 0f7b197ee4b3447ab59f1fffe84c7520(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:37,111 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:37,111 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/C, priority=13, startTime=1732005756276; duration=0sec 2024-11-19T08:42:37,111 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:37,111 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:C 2024-11-19T08:42:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005817116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005817116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005817116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005817246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,249 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005817325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005817326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005817326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T08:42:37,346 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:37,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-19T08:42:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T08:42:37,349 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:37,349 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:37,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:37,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:37,390 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111937b177a3d6004f7da21fb9d98990a751_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111937b177a3d6004f7da21fb9d98990a751_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:37,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/66f37fc8f01340bb839793371523f25b, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:37,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/66f37fc8f01340bb839793371523f25b is 175, key is test_row_0/A:col10/1732005756977/Put/seqid=0 2024-11-19T08:42:37,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742380_1556 (size=39749) 2024-11-19T08:42:37,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/66f37fc8f01340bb839793371523f25b 2024-11-19T08:42:37,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/77d0aeeb138c4698877e1ab5340f094d is 50, key is test_row_0/B:col10/1732005756977/Put/seqid=0 2024-11-19T08:42:37,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742381_1557 (size=12151) 2024-11-19T08:42:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T08:42:37,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T08:42:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005817635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005817636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005817636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:37,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T08:42:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T08:42:37,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/77d0aeeb138c4698877e1ab5340f094d 2024-11-19T08:42:37,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T08:42:37,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:37,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d7ed9fc9029b4ee7b11767bd939fff92 is 50, key is test_row_0/C:col10/1732005756977/Put/seqid=0 2024-11-19T08:42:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742382_1558 (size=12151) 2024-11-19T08:42:37,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T08:42:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:37,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T08:42:38,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T08:42:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:38,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T08:42:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005818146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005818146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005818146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:38,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d7ed9fc9029b4ee7b11767bd939fff92 2024-11-19T08:42:38,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/66f37fc8f01340bb839793371523f25b as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/66f37fc8f01340bb839793371523f25b 2024-11-19T08:42:38,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/66f37fc8f01340bb839793371523f25b, entries=200, sequenceid=212, filesize=38.8 K 2024-11-19T08:42:38,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/77d0aeeb138c4698877e1ab5340f094d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/77d0aeeb138c4698877e1ab5340f094d 2024-11-19T08:42:38,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/77d0aeeb138c4698877e1ab5340f094d, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T08:42:38,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/d7ed9fc9029b4ee7b11767bd939fff92 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d7ed9fc9029b4ee7b11767bd939fff92 2024-11-19T08:42:38,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d7ed9fc9029b4ee7b11767bd939fff92, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T08:42:38,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f0224c56755d95b541219e376c56cd39 in 1244ms, sequenceid=212, compaction requested=true 2024-11-19T08:42:38,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:38,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:38,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:38,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:38,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:38,222 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:38,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:38,223 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:38,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:38,223 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:38,223 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:38,223 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/B is initiating minor compaction (all files) 2024-11-19T08:42:38,223 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/A is initiating minor compaction (all files) 2024-11-19T08:42:38,223 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/A in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:38,223 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/B in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:38,223 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/aaf1c7d37c9c4144af8a22ffd5cc3ad9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/645ef749cc1247a0892129e5c80b4f23, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/77d0aeeb138c4698877e1ab5340f094d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=36.0 K 2024-11-19T08:42:38,223 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/52a229f07bfa45278e97917d6c9b715a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/510bcb2cd8844301ace60e1cbbfa5e48, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/66f37fc8f01340bb839793371523f25b] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=100.0 K 2024-11-19T08:42:38,224 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:38,224 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/52a229f07bfa45278e97917d6c9b715a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/510bcb2cd8844301ace60e1cbbfa5e48, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/66f37fc8f01340bb839793371523f25b] 2024-11-19T08:42:38,224 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf1c7d37c9c4144af8a22ffd5cc3ad9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732005754473 2024-11-19T08:42:38,224 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a229f07bfa45278e97917d6c9b715a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732005754473 2024-11-19T08:42:38,224 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 645ef749cc1247a0892129e5c80b4f23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732005755058 2024-11-19T08:42:38,224 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d0aeeb138c4698877e1ab5340f094d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005756655 2024-11-19T08:42:38,224 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510bcb2cd8844301ace60e1cbbfa5e48, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732005755058 2024-11-19T08:42:38,224 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66f37fc8f01340bb839793371523f25b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005756646 2024-11-19T08:42:38,228 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:38,229 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119d26bde6d156b46e28ef8f836d949d79f_f0224c56755d95b541219e376c56cd39 store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:38,230 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:38,231 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119d26bde6d156b46e28ef8f836d949d79f_f0224c56755d95b541219e376c56cd39, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:38,231 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/d54753bb3b14441294b514fe427f6d96 is 50, key is test_row_0/B:col10/1732005756977/Put/seqid=0 2024-11-19T08:42:38,231 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d26bde6d156b46e28ef8f836d949d79f_f0224c56755d95b541219e376c56cd39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:38,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742383_1559 (size=4469) 2024-11-19T08:42:38,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742384_1560 (size=12663) 2024-11-19T08:42:38,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T08:42:38,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:38,264 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T08:42:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dff135b093e24b239e32fa8b83b2966d_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005757005/Put/seqid=0 2024-11-19T08:42:38,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742385_1561 (size=12304) 2024-11-19T08:42:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T08:42:38,635 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:38,636 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/085dd143059f437b853f787ec9c88909 is 175, key is test_row_0/A:col10/1732005756977/Put/seqid=0 2024-11-19T08:42:38,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742386_1562 (size=31617) 2024-11-19T08:42:38,638 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/d54753bb3b14441294b514fe427f6d96 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d54753bb3b14441294b514fe427f6d96 2024-11-19T08:42:38,641 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/B of f0224c56755d95b541219e376c56cd39 into d54753bb3b14441294b514fe427f6d96(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:38,641 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:38,641 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/B, priority=13, startTime=1732005758222; duration=0sec 2024-11-19T08:42:38,641 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:38,641 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:B 2024-11-19T08:42:38,641 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:38,642 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:38,642 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/C is initiating minor compaction (all files) 2024-11-19T08:42:38,642 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/C in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:38,642 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0f7b197ee4b3447ab59f1fffe84c7520, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/df3979c8bb574a7c81461442e0035471, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d7ed9fc9029b4ee7b11767bd939fff92] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=36.0 K 2024-11-19T08:42:38,642 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7b197ee4b3447ab59f1fffe84c7520, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732005754473 2024-11-19T08:42:38,643 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting df3979c8bb574a7c81461442e0035471, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732005755058 2024-11-19T08:42:38,643 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ed9fc9029b4ee7b11767bd939fff92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005756655 2024-11-19T08:42:38,648 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:38,648 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/50632de3d3e844c9bb148ecf7e2b074c is 50, key is test_row_0/C:col10/1732005756977/Put/seqid=0 2024-11-19T08:42:38,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742387_1563 (size=12663) 2024-11-19T08:42:38,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:38,674 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dff135b093e24b239e32fa8b83b2966d_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dff135b093e24b239e32fa8b83b2966d_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:38,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/4f66e456cc7a41c08a85d7141c56db2e, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:38,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/4f66e456cc7a41c08a85d7141c56db2e is 175, key is test_row_0/A:col10/1732005757005/Put/seqid=0 2024-11-19T08:42:38,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:38,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742388_1564 (size=31105) 2024-11-19T08:42:38,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005818731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:38,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005818846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:39,042 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/085dd143059f437b853f787ec9c88909 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/085dd143059f437b853f787ec9c88909 2024-11-19T08:42:39,046 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/A of f0224c56755d95b541219e376c56cd39 into 085dd143059f437b853f787ec9c88909(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:39,046 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:39,046 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/A, priority=13, startTime=1732005758222; duration=0sec 2024-11-19T08:42:39,046 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:39,046 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:A 2024-11-19T08:42:39,057 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/50632de3d3e844c9bb148ecf7e2b074c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/50632de3d3e844c9bb148ecf7e2b074c 2024-11-19T08:42:39,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005819056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:39,060 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/C of f0224c56755d95b541219e376c56cd39 into 50632de3d3e844c9bb148ecf7e2b074c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:39,060 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:39,060 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/C, priority=13, startTime=1732005758222; duration=0sec 2024-11-19T08:42:39,060 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:39,060 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:C 2024-11-19T08:42:39,080 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/4f66e456cc7a41c08a85d7141c56db2e 2024-11-19T08:42:39,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/dae84dc8d20d464ba48c1e8ed2fb5b70 is 50, key is test_row_0/B:col10/1732005757005/Put/seqid=0 2024-11-19T08:42:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742389_1565 (size=12151) 2024-11-19T08:42:39,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/dae84dc8d20d464ba48c1e8ed2fb5b70 2024-11-19T08:42:39,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/5cf712a0565c4d84ba789d000f88fe77 is 50, key is test_row_0/C:col10/1732005757005/Put/seqid=0 2024-11-19T08:42:39,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742390_1566 (size=12151) 2024-11-19T08:42:39,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005819155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:39,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005819156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:39,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005819156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:39,158 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:39,159 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:39,159 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45056 deadline: 1732005819266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:39,268 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005819365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T08:42:39,497 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/5cf712a0565c4d84ba789d000f88fe77 2024-11-19T08:42:39,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/4f66e456cc7a41c08a85d7141c56db2e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/4f66e456cc7a41c08a85d7141c56db2e 2024-11-19T08:42:39,512 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/4f66e456cc7a41c08a85d7141c56db2e, entries=150, sequenceid=232, filesize=30.4 K 2024-11-19T08:42:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/dae84dc8d20d464ba48c1e8ed2fb5b70 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/dae84dc8d20d464ba48c1e8ed2fb5b70 2024-11-19T08:42:39,517 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/dae84dc8d20d464ba48c1e8ed2fb5b70, entries=150, sequenceid=232, filesize=11.9 K 2024-11-19T08:42:39,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/5cf712a0565c4d84ba789d000f88fe77 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5cf712a0565c4d84ba789d000f88fe77 2024-11-19T08:42:39,520 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5cf712a0565c4d84ba789d000f88fe77, entries=150, sequenceid=232, filesize=11.9 K 2024-11-19T08:42:39,521 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f0224c56755d95b541219e376c56cd39 in 1257ms, sequenceid=232, compaction requested=false 2024-11-19T08:42:39,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:39,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:39,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-19T08:42:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-19T08:42:39,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-19T08:42:39,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-19T08:42:39,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.1800 sec 2024-11-19T08:42:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:39,878 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:39,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T08:42:39,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196efaabda90ed4604a2f357b433e09c8e_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005758730/Put/seqid=0 2024-11-19T08:42:39,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742391_1567 (size=14794) 2024-11-19T08:42:39,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:39,894 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196efaabda90ed4604a2f357b433e09c8e_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196efaabda90ed4604a2f357b433e09c8e_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:39,895 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/69ef79e188544786bfd0b406b16aac47, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:39,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/69ef79e188544786bfd0b406b16aac47 is 175, key is test_row_0/A:col10/1732005758730/Put/seqid=0 2024-11-19T08:42:39,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742392_1568 (size=39749) 2024-11-19T08:42:39,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/69ef79e188544786bfd0b406b16aac47 2024-11-19T08:42:39,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/6bcb96c4f999487aa9221de0f14079ae is 50, key is test_row_0/B:col10/1732005758730/Put/seqid=0 2024-11-19T08:42:39,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742393_1569 (size=12151) 2024-11-19T08:42:39,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005819940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:39,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:40,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005820046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:40,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:40,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:40,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005820256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:40,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/6bcb96c4f999487aa9221de0f14079ae 2024-11-19T08:42:40,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/6380d86c93e649438165050d797fcc4a is 50, key is test_row_0/C:col10/1732005758730/Put/seqid=0 2024-11-19T08:42:40,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742394_1570 (size=12151) 2024-11-19T08:42:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005820566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:40,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/6380d86c93e649438165050d797fcc4a 2024-11-19T08:42:40,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/69ef79e188544786bfd0b406b16aac47 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/69ef79e188544786bfd0b406b16aac47 2024-11-19T08:42:40,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/69ef79e188544786bfd0b406b16aac47, entries=200, sequenceid=253, filesize=38.8 K 2024-11-19T08:42:40,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/6bcb96c4f999487aa9221de0f14079ae as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6bcb96c4f999487aa9221de0f14079ae 2024-11-19T08:42:40,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6bcb96c4f999487aa9221de0f14079ae, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T08:42:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/6380d86c93e649438165050d797fcc4a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6380d86c93e649438165050d797fcc4a 2024-11-19T08:42:40,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6380d86c93e649438165050d797fcc4a, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T08:42:40,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for f0224c56755d95b541219e376c56cd39 in 862ms, sequenceid=253, compaction requested=true 2024-11-19T08:42:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:40,740 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:40,740 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:40,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:40,741 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:40,741 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:40,741 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/A is initiating minor compaction (all files) 2024-11-19T08:42:40,741 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/B is initiating minor compaction (all files) 2024-11-19T08:42:40,741 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/A in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:40,741 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/B in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:40,741 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d54753bb3b14441294b514fe427f6d96, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/dae84dc8d20d464ba48c1e8ed2fb5b70, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6bcb96c4f999487aa9221de0f14079ae] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=36.1 K 2024-11-19T08:42:40,741 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/085dd143059f437b853f787ec9c88909, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/4f66e456cc7a41c08a85d7141c56db2e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/69ef79e188544786bfd0b406b16aac47] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=100.1 K 2024-11-19T08:42:40,742 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:40,742 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting d54753bb3b14441294b514fe427f6d96, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005756655 2024-11-19T08:42:40,742 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/085dd143059f437b853f787ec9c88909, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/4f66e456cc7a41c08a85d7141c56db2e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/69ef79e188544786bfd0b406b16aac47] 2024-11-19T08:42:40,742 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting dae84dc8d20d464ba48c1e8ed2fb5b70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732005757003 2024-11-19T08:42:40,742 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085dd143059f437b853f787ec9c88909, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005756655 2024-11-19T08:42:40,742 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bcb96c4f999487aa9221de0f14079ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732005758707 2024-11-19T08:42:40,742 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f66e456cc7a41c08a85d7141c56db2e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732005757003 2024-11-19T08:42:40,742 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69ef79e188544786bfd0b406b16aac47, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732005758707 2024-11-19T08:42:40,747 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:40,747 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/1c584e6c9a5f49049326b0b6cc022b78 is 50, key is test_row_0/B:col10/1732005758730/Put/seqid=0 2024-11-19T08:42:40,749 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:40,750 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119fae5c95f696140bc8ab29f5bf3f802c8_f0224c56755d95b541219e376c56cd39 store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:40,751 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119fae5c95f696140bc8ab29f5bf3f802c8_f0224c56755d95b541219e376c56cd39, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:40,751 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fae5c95f696140bc8ab29f5bf3f802c8_f0224c56755d95b541219e376c56cd39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:40,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742395_1571 (size=12765) 2024-11-19T08:42:40,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742396_1572 (size=4469) 2024-11-19T08:42:40,756 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#A#compaction#481 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:40,756 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a is 175, key is test_row_0/A:col10/1732005758730/Put/seqid=0 2024-11-19T08:42:40,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742397_1573 (size=31719) 2024-11-19T08:42:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:41,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:41,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T08:42:41,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a2602fd4243d41b99160aee425e36b5b_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005759939/Put/seqid=0 2024-11-19T08:42:41,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742398_1574 (size=14994) 2024-11-19T08:42:41,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:41,090 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a2602fd4243d41b99160aee425e36b5b_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2602fd4243d41b99160aee425e36b5b_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:41,091 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/6d4e8bf920d343ef9ffec2f07da9cd70, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:41,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/6d4e8bf920d343ef9ffec2f07da9cd70 is 175, key is test_row_0/A:col10/1732005759939/Put/seqid=0 2024-11-19T08:42:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742399_1575 (size=39949) 2024-11-19T08:42:41,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005821125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:41,156 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/1c584e6c9a5f49049326b0b6cc022b78 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/1c584e6c9a5f49049326b0b6cc022b78 2024-11-19T08:42:41,159 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/B of f0224c56755d95b541219e376c56cd39 into 1c584e6c9a5f49049326b0b6cc022b78(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:41,159 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:41,159 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/B, priority=13, startTime=1732005760740; duration=0sec 2024-11-19T08:42:41,159 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:41,159 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:B 2024-11-19T08:42:41,159 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T08:42:41,161 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T08:42:41,161 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/C is initiating minor compaction (all files) 2024-11-19T08:42:41,161 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/C in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:41,162 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/50632de3d3e844c9bb148ecf7e2b074c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5cf712a0565c4d84ba789d000f88fe77, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6380d86c93e649438165050d797fcc4a] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=36.1 K 2024-11-19T08:42:41,162 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 50632de3d3e844c9bb148ecf7e2b074c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732005756655 2024-11-19T08:42:41,162 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf712a0565c4d84ba789d000f88fe77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732005757003 2024-11-19T08:42:41,162 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 6380d86c93e649438165050d797fcc4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732005758707 2024-11-19T08:42:41,174 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a 2024-11-19T08:42:41,177 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:41,178 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/cf42f9f3760b428d8217e777b838e54c is 50, key is test_row_0/C:col10/1732005758730/Put/seqid=0 2024-11-19T08:42:41,178 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/A of f0224c56755d95b541219e376c56cd39 into b40dd51f815d4dd58c6cd2fe7fe4ec8a(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:41,178 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:41,178 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/A, priority=13, startTime=1732005760740; duration=0sec 2024-11-19T08:42:41,178 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:41,178 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:A 2024-11-19T08:42:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45074 deadline: 1732005821176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45110 deadline: 1732005821176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,180 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:41,180 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_1' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:41,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742400_1576 (size=12765) 2024-11-19T08:42:41,186 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/cf42f9f3760b428d8217e777b838e54c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/cf42f9f3760b428d8217e777b838e54c 2024-11-19T08:42:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45094 deadline: 1732005821186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,189 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3ab37fa97a98,34165,1732005593269 for 'test_row_2' in f0224c56755d95b541219e376c56cd39 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T08:42:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:41,190 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0224c56755d95b541219e376c56cd39/C of f0224c56755d95b541219e376c56cd39 into cf42f9f3760b428d8217e777b838e54c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:41,190 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:41,190 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/C, priority=13, startTime=1732005760740; duration=0sec 2024-11-19T08:42:41,191 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:41,191 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:C 2024-11-19T08:42:41,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005821236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:41,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005821446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:41,494 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/6d4e8bf920d343ef9ffec2f07da9cd70 2024-11-19T08:42:41,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/29de1e1412834177a0dc1783d3169644 is 50, key is test_row_0/B:col10/1732005759939/Put/seqid=0 2024-11-19T08:42:41,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742401_1577 (size=12301) 2024-11-19T08:42:41,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/29de1e1412834177a0dc1783d3169644 2024-11-19T08:42:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T08:42:41,507 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:41,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T08:42:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-19T08:42:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T08:42:41,508 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T08:42:41,509 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T08:42:41,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T08:42:41,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/f2a1ad76243549479e5ffe2da342f2dd is 50, key is test_row_0/C:col10/1732005759939/Put/seqid=0 2024-11-19T08:42:41,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742402_1578 (size=12301) 2024-11-19T08:42:41,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/f2a1ad76243549479e5ffe2da342f2dd 2024-11-19T08:42:41,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/6d4e8bf920d343ef9ffec2f07da9cd70 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/6d4e8bf920d343ef9ffec2f07da9cd70 2024-11-19T08:42:41,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/6d4e8bf920d343ef9ffec2f07da9cd70, entries=200, sequenceid=272, filesize=39.0 K 2024-11-19T08:42:41,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/29de1e1412834177a0dc1783d3169644 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/29de1e1412834177a0dc1783d3169644 2024-11-19T08:42:41,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/29de1e1412834177a0dc1783d3169644, entries=150, sequenceid=272, filesize=12.0 K 2024-11-19T08:42:41,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/f2a1ad76243549479e5ffe2da342f2dd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f2a1ad76243549479e5ffe2da342f2dd 2024-11-19T08:42:41,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f2a1ad76243549479e5ffe2da342f2dd, entries=150, sequenceid=272, filesize=12.0 K 2024-11-19T08:42:41,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for f0224c56755d95b541219e376c56cd39 in 450ms, sequenceid=272, compaction requested=false 2024-11-19T08:42:41,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T08:42:41,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34165 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T08:42:41,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:41,662 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T08:42:41,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119afcddc58bb8b4b19b0fbe97c3e663a03_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005761124/Put/seqid=0 2024-11-19T08:42:41,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742403_1579 (size=12454) 2024-11-19T08:42:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:41,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. as already flushing 2024-11-19T08:42:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T08:42:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005821845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:41,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005821956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:42,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:42,081 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119afcddc58bb8b4b19b0fbe97c3e663a03_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afcddc58bb8b4b19b0fbe97c3e663a03_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad8efe412dae4ee8bf18ba8c7e23fbba, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad8efe412dae4ee8bf18ba8c7e23fbba is 175, key is test_row_0/A:col10/1732005761124/Put/seqid=0 2024-11-19T08:42:42,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742404_1580 (size=31255) 2024-11-19T08:42:42,097 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad8efe412dae4ee8bf18ba8c7e23fbba 2024-11-19T08:42:42,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/b20799ce03f24a16bb6f2e26b91454bb is 50, key is test_row_0/B:col10/1732005761124/Put/seqid=0 2024-11-19T08:42:42,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742405_1581 (size=12301) 2024-11-19T08:42:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T08:42:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005822166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:42,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T08:42:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1732005822476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 , the old value is region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T08:42:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., hostname=3ab37fa97a98,34165,1732005593269, seqNum=5 because the exception is null or not the one we care about 2024-11-19T08:42:42,528 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/b20799ce03f24a16bb6f2e26b91454bb 2024-11-19T08:42:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0af967426a994e0e87856db34b54b29e is 50, key is test_row_0/C:col10/1732005761124/Put/seqid=0 2024-11-19T08:42:42,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742406_1582 (size=12301) 2024-11-19T08:42:42,555 INFO [Thread-2251 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2251. 2024-11-19T08:42:42,555 DEBUG [Thread-2251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:42,555 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,556 DEBUG [Thread-2251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:42,556 INFO [Thread-2247 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2247. 2024-11-19T08:42:42,557 DEBUG [Thread-2247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:42,557 DEBUG [Thread-2247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,557 DEBUG [Thread-2247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:42,559 INFO [Thread-2245 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2245. 2024-11-19T08:42:42,559 DEBUG [Thread-2245 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:42,559 DEBUG [Thread-2245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,559 DEBUG [Thread-2245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:42,560 INFO [Thread-2249 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2249. 2024-11-19T08:42:42,560 DEBUG [Thread-2249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:42,560 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,560 DEBUG [Thread-2249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:42,560 INFO [Thread-2243 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2243. 2024-11-19T08:42:42,560 DEBUG [Thread-2243 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:42,560 DEBUG [Thread-2243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,560 DEBUG [Thread-2243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T08:42:42,937 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0af967426a994e0e87856db34b54b29e 2024-11-19T08:42:42,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad8efe412dae4ee8bf18ba8c7e23fbba as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad8efe412dae4ee8bf18ba8c7e23fbba 2024-11-19T08:42:42,951 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad8efe412dae4ee8bf18ba8c7e23fbba, entries=150, sequenceid=292, filesize=30.5 K 2024-11-19T08:42:42,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/b20799ce03f24a16bb6f2e26b91454bb as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b20799ce03f24a16bb6f2e26b91454bb 2024-11-19T08:42:42,955 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b20799ce03f24a16bb6f2e26b91454bb, entries=150, sequenceid=292, filesize=12.0 K 2024-11-19T08:42:42,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0af967426a994e0e87856db34b54b29e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0af967426a994e0e87856db34b54b29e 2024-11-19T08:42:42,959 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0af967426a994e0e87856db34b54b29e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-19T08:42:42,960 INFO [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f0224c56755d95b541219e376c56cd39 in 1299ms, sequenceid=292, compaction requested=true 2024-11-19T08:42:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:42,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3ab37fa97a98:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-19T08:42:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-19T08:42:42,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-19T08:42:42,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4520 sec 2024-11-19T08:42:42,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.4560 sec 2024-11-19T08:42:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34165 {}] regionserver.HRegion(8855): Flush requested on f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:42,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T08:42:42,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T08:42:42,988 INFO [Thread-2236 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2236. 2024-11-19T08:42:42,988 DEBUG [Thread-2236 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:42,989 DEBUG [Thread-2236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,989 DEBUG [Thread-2236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:42,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:42,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119762e9380b118459ca04e4b3575e5dbf0_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_0/A:col10/1732005762987/Put/seqid=0 2024-11-19T08:42:43,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742407_1583 (size=12454) 2024-11-19T08:42:43,311 INFO [Thread-2234 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2234. 2024-11-19T08:42:43,312 DEBUG [Thread-2234 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:43,312 DEBUG [Thread-2234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:43,312 DEBUG [Thread-2234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:43,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:43,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:43,413 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119762e9380b118459ca04e4b3575e5dbf0_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119762e9380b118459ca04e4b3575e5dbf0_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:43,414 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad9dd0adbeb64afca0ef83308439f96d, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:43,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad9dd0adbeb64afca0ef83308439f96d is 175, key is test_row_0/A:col10/1732005762987/Put/seqid=0 2024-11-19T08:42:43,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742408_1584 (size=31255) 2024-11-19T08:42:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T08:42:43,658 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:43,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad9dd0adbeb64afca0ef83308439f96d 2024-11-19T08:42:43,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/feb8c8f1314d4b14a851e12d54df56bd is 50, key is test_row_0/B:col10/1732005762987/Put/seqid=0 2024-11-19T08:42:43,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742409_1585 (size=12301) 2024-11-19T08:42:44,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/feb8c8f1314d4b14a851e12d54df56bd 2024-11-19T08:42:44,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0cdf6c8c02ac423ca209afe064fffef6 is 50, key is test_row_0/C:col10/1732005762987/Put/seqid=0 2024-11-19T08:42:44,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742410_1586 (size=12301) 2024-11-19T08:42:44,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0cdf6c8c02ac423ca209afe064fffef6 2024-11-19T08:42:44,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/ad9dd0adbeb64afca0ef83308439f96d as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad9dd0adbeb64afca0ef83308439f96d 2024-11-19T08:42:44,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad9dd0adbeb64afca0ef83308439f96d, entries=150, sequenceid=312, filesize=30.5 K 2024-11-19T08:42:44,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/feb8c8f1314d4b14a851e12d54df56bd as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/feb8c8f1314d4b14a851e12d54df56bd 2024-11-19T08:42:44,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/feb8c8f1314d4b14a851e12d54df56bd, entries=150, sequenceid=312, filesize=12.0 K 2024-11-19T08:42:44,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/0cdf6c8c02ac423ca209afe064fffef6 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cdf6c8c02ac423ca209afe064fffef6 2024-11-19T08:42:44,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cdf6c8c02ac423ca209afe064fffef6, entries=150, sequenceid=312, filesize=12.0 K 2024-11-19T08:42:44,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for f0224c56755d95b541219e376c56cd39 in 1697ms, sequenceid=312, compaction requested=true 2024-11-19T08:42:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T08:42:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T08:42:44,685 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:42:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0224c56755d95b541219e376c56cd39:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T08:42:44,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:44,685 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:42:44,686 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:42:44,686 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134178 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:42:44,686 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/B is initiating minor compaction (all files) 2024-11-19T08:42:44,686 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/A is initiating minor compaction (all files) 2024-11-19T08:42:44,687 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/B in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:44,687 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/A in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:44,687 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/1c584e6c9a5f49049326b0b6cc022b78, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/29de1e1412834177a0dc1783d3169644, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b20799ce03f24a16bb6f2e26b91454bb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/feb8c8f1314d4b14a851e12d54df56bd] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=48.5 K 2024-11-19T08:42:44,687 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/6d4e8bf920d343ef9ffec2f07da9cd70, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad8efe412dae4ee8bf18ba8c7e23fbba, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad9dd0adbeb64afca0ef83308439f96d] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=131.0 K 2024-11-19T08:42:44,687 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:44,687 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. files: [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/6d4e8bf920d343ef9ffec2f07da9cd70, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad8efe412dae4ee8bf18ba8c7e23fbba, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad9dd0adbeb64afca0ef83308439f96d] 2024-11-19T08:42:44,687 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c584e6c9a5f49049326b0b6cc022b78, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732005758707 2024-11-19T08:42:44,687 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40dd51f815d4dd58c6cd2fe7fe4ec8a, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732005758707 2024-11-19T08:42:44,687 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 29de1e1412834177a0dc1783d3169644, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732005759935 2024-11-19T08:42:44,688 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d4e8bf920d343ef9ffec2f07da9cd70, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732005759908 2024-11-19T08:42:44,688 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting b20799ce03f24a16bb6f2e26b91454bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732005761116 2024-11-19T08:42:44,688 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad8efe412dae4ee8bf18ba8c7e23fbba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732005761116 2024-11-19T08:42:44,688 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting feb8c8f1314d4b14a851e12d54df56bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732005761838 2024-11-19T08:42:44,688 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9dd0adbeb64afca0ef83308439f96d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732005761838 2024-11-19T08:42:44,697 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:44,699 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#B#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:44,699 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119ec126190ebb949d9bef81b9882cec3b1_f0224c56755d95b541219e376c56cd39 store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:44,700 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/393a7df34c75419cafc47786ec0d3b55 is 50, key is test_row_0/B:col10/1732005762987/Put/seqid=0 2024-11-19T08:42:44,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742411_1587 (size=13051) 2024-11-19T08:42:44,707 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119ec126190ebb949d9bef81b9882cec3b1_f0224c56755d95b541219e376c56cd39, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:44,707 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ec126190ebb949d9bef81b9882cec3b1_f0224c56755d95b541219e376c56cd39 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:44,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742412_1588 (size=4469) 2024-11-19T08:42:45,113 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:45,114 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/e7cb9a65b2e8478197712a52d90b4f69 is 175, key is test_row_0/A:col10/1732005762987/Put/seqid=0 2024-11-19T08:42:45,116 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/393a7df34c75419cafc47786ec0d3b55 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/393a7df34c75419cafc47786ec0d3b55 2024-11-19T08:42:45,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742413_1589 (size=32005) 2024-11-19T08:42:45,122 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0224c56755d95b541219e376c56cd39/B of f0224c56755d95b541219e376c56cd39 into 393a7df34c75419cafc47786ec0d3b55(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:45,122 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:45,122 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/B, priority=12, startTime=1732005764685; duration=0sec 2024-11-19T08:42:45,122 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T08:42:45,122 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:B 2024-11-19T08:42:45,122 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T08:42:45,123 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T08:42:45,123 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1541): f0224c56755d95b541219e376c56cd39/C is initiating minor compaction (all files) 2024-11-19T08:42:45,123 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0224c56755d95b541219e376c56cd39/C in TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:45,123 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/cf42f9f3760b428d8217e777b838e54c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f2a1ad76243549479e5ffe2da342f2dd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0af967426a994e0e87856db34b54b29e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cdf6c8c02ac423ca209afe064fffef6] into tmpdir=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp, totalSize=48.5 K 2024-11-19T08:42:45,123 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting cf42f9f3760b428d8217e777b838e54c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732005758707 2024-11-19T08:42:45,124 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a1ad76243549479e5ffe2da342f2dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732005759935 2024-11-19T08:42:45,124 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af967426a994e0e87856db34b54b29e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732005761116 2024-11-19T08:42:45,124 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cdf6c8c02ac423ca209afe064fffef6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732005761838 2024-11-19T08:42:45,131 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0224c56755d95b541219e376c56cd39#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T08:42:45,131 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/e0080e0c2de24648aa6c30d2c3b690d4 is 50, key is test_row_0/C:col10/1732005762987/Put/seqid=0 2024-11-19T08:42:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742414_1590 (size=13051) 2024-11-19T08:42:45,199 INFO [Thread-2238 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2238. 2024-11-19T08:42:45,199 DEBUG [Thread-2238 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:45,199 DEBUG [Thread-2238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,199 DEBUG [Thread-2238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:45,217 INFO [Thread-2232 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2232. 2024-11-19T08:42:45,217 INFO [Thread-2240 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2240. 2024-11-19T08:42:45,217 DEBUG [Thread-2240 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:45,217 DEBUG [Thread-2232 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T08:42:45,217 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,217 DEBUG [Thread-2232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,217 DEBUG [Thread-2240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,217 DEBUG [Thread-2232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:45,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3850 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3732 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3694 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3841 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3719 2024-11-19T08:42:45,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T08:42:45,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:42:45,217 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:42:45,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:45,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:45,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T08:42:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T08:42:45,221 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005765221"}]},"ts":"1732005765221"} 2024-11-19T08:42:45,222 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T08:42:45,222 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T08:42:45,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T08:42:45,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, UNASSIGN}] 2024-11-19T08:42:45,225 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, UNASSIGN 2024-11-19T08:42:45,226 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=CLOSING, regionLocation=3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:45,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, UNASSIGN because future has completed 2024-11-19T08:42:45,228 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T08:42:45,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269}] 2024-11-19T08:42:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T08:42:45,381 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:45,381 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T08:42:45,381 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing f0224c56755d95b541219e376c56cd39, disabling compactions & flushes 2024-11-19T08:42:45,381 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:45,525 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/e7cb9a65b2e8478197712a52d90b4f69 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e7cb9a65b2e8478197712a52d90b4f69 2024-11-19T08:42:45,533 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0224c56755d95b541219e376c56cd39/A of f0224c56755d95b541219e376c56cd39 into e7cb9a65b2e8478197712a52d90b4f69(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:45,533 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:45,533 INFO [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/A, priority=12, startTime=1732005764685; duration=0sec 2024-11-19T08:42:45,533 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:45,533 DEBUG [RS:0;3ab37fa97a98:34165-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:A 2024-11-19T08:42:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T08:42:45,539 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/e0080e0c2de24648aa6c30d2c3b690d4 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/e0080e0c2de24648aa6c30d2c3b690d4 2024-11-19T08:42:45,542 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0224c56755d95b541219e376c56cd39/C of f0224c56755d95b541219e376c56cd39 into e0080e0c2de24648aa6c30d2c3b690d4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T08:42:45,543 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0224c56755d95b541219e376c56cd39: 2024-11-19T08:42:45,543 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:45,543 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:45,543 INFO [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., storeName=f0224c56755d95b541219e376c56cd39/C, priority=12, startTime=1732005764685; duration=0sec 2024-11-19T08:42:45,543 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. after waiting 0 ms 2024-11-19T08:42:45,543 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:45,543 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T08:42:45,543 DEBUG [RS:0;3ab37fa97a98:34165-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0224c56755d95b541219e376c56cd39:C 2024-11-19T08:42:45,543 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing f0224c56755d95b541219e376c56cd39 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-19T08:42:45,547 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119667e13ba63a34633950f072eea2f11b0_f0224c56755d95b541219e376c56cd39 is 50, key is test_row_1/A:col10/1732005765216/Put/seqid=0 2024-11-19T08:42:45,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742415_1591 (size=9914) 2024-11-19T08:42:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T08:42:45,950 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T08:42:45,957 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119667e13ba63a34633950f072eea2f11b0_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119667e13ba63a34633950f072eea2f11b0_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:45,958 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/c7d4051aac3d4ff386069d89f74806a2, store: [table=TestAcidGuarantees family=A region=f0224c56755d95b541219e376c56cd39] 2024-11-19T08:42:45,959 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/c7d4051aac3d4ff386069d89f74806a2 is 175, key is test_row_1/A:col10/1732005765216/Put/seqid=0 2024-11-19T08:42:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742416_1592 (size=22561) 2024-11-19T08:42:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T08:42:46,363 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/c7d4051aac3d4ff386069d89f74806a2 2024-11-19T08:42:46,369 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/575151f9e3fe45ec8bbf83e045144e4f is 50, key is test_row_1/B:col10/1732005765216/Put/seqid=0 2024-11-19T08:42:46,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742417_1593 (size=9857) 2024-11-19T08:42:46,773 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/575151f9e3fe45ec8bbf83e045144e4f 2024-11-19T08:42:46,785 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/8bb29edac3b745cb8276cf09877a3151 is 50, key is test_row_1/C:col10/1732005765216/Put/seqid=0 2024-11-19T08:42:46,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742418_1594 (size=9857) 2024-11-19T08:42:47,202 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/8bb29edac3b745cb8276cf09877a3151 2024-11-19T08:42:47,214 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/A/c7d4051aac3d4ff386069d89f74806a2 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/c7d4051aac3d4ff386069d89f74806a2 2024-11-19T08:42:47,217 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/c7d4051aac3d4ff386069d89f74806a2, entries=100, sequenceid=322, filesize=22.0 K 2024-11-19T08:42:47,218 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/B/575151f9e3fe45ec8bbf83e045144e4f as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/575151f9e3fe45ec8bbf83e045144e4f 2024-11-19T08:42:47,221 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/575151f9e3fe45ec8bbf83e045144e4f, entries=100, sequenceid=322, filesize=9.6 K 2024-11-19T08:42:47,221 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/.tmp/C/8bb29edac3b745cb8276cf09877a3151 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/8bb29edac3b745cb8276cf09877a3151 2024-11-19T08:42:47,224 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/8bb29edac3b745cb8276cf09877a3151, entries=100, sequenceid=322, filesize=9.6 K 2024-11-19T08:42:47,225 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f0224c56755d95b541219e376c56cd39 in 1681ms, sequenceid=322, compaction requested=false 2024-11-19T08:42:47,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/af2a539d9c594f39be75e749b6a081da, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/0e3edfd7a3ec46fe83de9c0c5930a430, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/615df182c4bc4f8db14ee3c8b711b1d6, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/bd1e358adc064d5a8f5027ce640d2617, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3defcc68058f493fbbecfe586c25539d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d2559b11f0704a00a7c514a36e218b10, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e74087b6d5194dbb99d84f0d49c500dc, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ef2bf1fc612e46a99cdd61d54c7d85e5, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3cc71816cd274087b906e9540d3001d8, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b5d85d69bcfe49158442262717c2aa80, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d192a8d73b894e52bfb771c6a837ea3d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/787a2509e39b4d999a6cce049ccd7716, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/52a229f07bfa45278e97917d6c9b715a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/510bcb2cd8844301ace60e1cbbfa5e48, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/66f37fc8f01340bb839793371523f25b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/085dd143059f437b853f787ec9c88909, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/4f66e456cc7a41c08a85d7141c56db2e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/69ef79e188544786bfd0b406b16aac47, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/6d4e8bf920d343ef9ffec2f07da9cd70, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad8efe412dae4ee8bf18ba8c7e23fbba, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad9dd0adbeb64afca0ef83308439f96d] to archive 2024-11-19T08:42:47,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:42:47,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/af2a539d9c594f39be75e749b6a081da to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/af2a539d9c594f39be75e749b6a081da 2024-11-19T08:42:47,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/0e3edfd7a3ec46fe83de9c0c5930a430 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/0e3edfd7a3ec46fe83de9c0c5930a430 2024-11-19T08:42:47,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/615df182c4bc4f8db14ee3c8b711b1d6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/615df182c4bc4f8db14ee3c8b711b1d6 2024-11-19T08:42:47,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/bd1e358adc064d5a8f5027ce640d2617 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/bd1e358adc064d5a8f5027ce640d2617 2024-11-19T08:42:47,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3defcc68058f493fbbecfe586c25539d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3defcc68058f493fbbecfe586c25539d 2024-11-19T08:42:47,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d2559b11f0704a00a7c514a36e218b10 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d2559b11f0704a00a7c514a36e218b10 2024-11-19T08:42:47,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e74087b6d5194dbb99d84f0d49c500dc to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e74087b6d5194dbb99d84f0d49c500dc 2024-11-19T08:42:47,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ef2bf1fc612e46a99cdd61d54c7d85e5 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ef2bf1fc612e46a99cdd61d54c7d85e5 2024-11-19T08:42:47,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3cc71816cd274087b906e9540d3001d8 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/3cc71816cd274087b906e9540d3001d8 2024-11-19T08:42:47,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b5d85d69bcfe49158442262717c2aa80 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b5d85d69bcfe49158442262717c2aa80 2024-11-19T08:42:47,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d192a8d73b894e52bfb771c6a837ea3d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/d192a8d73b894e52bfb771c6a837ea3d 2024-11-19T08:42:47,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/787a2509e39b4d999a6cce049ccd7716 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/787a2509e39b4d999a6cce049ccd7716 2024-11-19T08:42:47,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/52a229f07bfa45278e97917d6c9b715a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/52a229f07bfa45278e97917d6c9b715a 2024-11-19T08:42:47,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/510bcb2cd8844301ace60e1cbbfa5e48 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/510bcb2cd8844301ace60e1cbbfa5e48 2024-11-19T08:42:47,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/66f37fc8f01340bb839793371523f25b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/66f37fc8f01340bb839793371523f25b 2024-11-19T08:42:47,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/085dd143059f437b853f787ec9c88909 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/085dd143059f437b853f787ec9c88909 2024-11-19T08:42:47,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/4f66e456cc7a41c08a85d7141c56db2e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/4f66e456cc7a41c08a85d7141c56db2e 2024-11-19T08:42:47,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/69ef79e188544786bfd0b406b16aac47 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/69ef79e188544786bfd0b406b16aac47 2024-11-19T08:42:47,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/b40dd51f815d4dd58c6cd2fe7fe4ec8a 2024-11-19T08:42:47,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/6d4e8bf920d343ef9ffec2f07da9cd70 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/6d4e8bf920d343ef9ffec2f07da9cd70 2024-11-19T08:42:47,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad8efe412dae4ee8bf18ba8c7e23fbba to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad8efe412dae4ee8bf18ba8c7e23fbba 2024-11-19T08:42:47,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad9dd0adbeb64afca0ef83308439f96d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/ad9dd0adbeb64afca0ef83308439f96d 2024-11-19T08:42:47,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a2e7d54f3604496bbcf7d09f845e8671, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/f37a136c56534424a82941475a24b5e9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b75f73f533b3401ab3e82b38a814db6c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/e5d8a224b0bc4858a4c0cf7e509e94fe, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/cca72c9f31f648ff9ccf24d6747c8c52, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d0d361f9ed614f52b5676ce312618a5a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6adf7e8ac0ac4005b814f67d160c9b05, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a9e1193dcef140fb905c986ca635c85b, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/5cdf8fa4d07149cc99a99b04bafe6cad, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d2028bea642946dc9fcdd3f25886d29f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/2f2b7b3ad6024b738baacd2f09de68cb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/aaf1c7d37c9c4144af8a22ffd5cc3ad9, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/ace99d3f297b46e7b472f22fe99559f1, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/645ef749cc1247a0892129e5c80b4f23, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d54753bb3b14441294b514fe427f6d96, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/77d0aeeb138c4698877e1ab5340f094d, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/dae84dc8d20d464ba48c1e8ed2fb5b70, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/1c584e6c9a5f49049326b0b6cc022b78, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6bcb96c4f999487aa9221de0f14079ae, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/29de1e1412834177a0dc1783d3169644, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b20799ce03f24a16bb6f2e26b91454bb, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/feb8c8f1314d4b14a851e12d54df56bd] to archive 2024-11-19T08:42:47,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:42:47,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a2e7d54f3604496bbcf7d09f845e8671 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a2e7d54f3604496bbcf7d09f845e8671 2024-11-19T08:42:47,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/f37a136c56534424a82941475a24b5e9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/f37a136c56534424a82941475a24b5e9 2024-11-19T08:42:47,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b75f73f533b3401ab3e82b38a814db6c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b75f73f533b3401ab3e82b38a814db6c 2024-11-19T08:42:47,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/e5d8a224b0bc4858a4c0cf7e509e94fe to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/e5d8a224b0bc4858a4c0cf7e509e94fe 2024-11-19T08:42:47,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/cca72c9f31f648ff9ccf24d6747c8c52 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/cca72c9f31f648ff9ccf24d6747c8c52 2024-11-19T08:42:47,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d0d361f9ed614f52b5676ce312618a5a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d0d361f9ed614f52b5676ce312618a5a 2024-11-19T08:42:47,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6adf7e8ac0ac4005b814f67d160c9b05 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6adf7e8ac0ac4005b814f67d160c9b05 2024-11-19T08:42:47,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a9e1193dcef140fb905c986ca635c85b to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/a9e1193dcef140fb905c986ca635c85b 2024-11-19T08:42:47,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/5cdf8fa4d07149cc99a99b04bafe6cad to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/5cdf8fa4d07149cc99a99b04bafe6cad 2024-11-19T08:42:47,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d2028bea642946dc9fcdd3f25886d29f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d2028bea642946dc9fcdd3f25886d29f 2024-11-19T08:42:47,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/2f2b7b3ad6024b738baacd2f09de68cb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/2f2b7b3ad6024b738baacd2f09de68cb 2024-11-19T08:42:47,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/aaf1c7d37c9c4144af8a22ffd5cc3ad9 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/aaf1c7d37c9c4144af8a22ffd5cc3ad9 2024-11-19T08:42:47,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/ace99d3f297b46e7b472f22fe99559f1 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/ace99d3f297b46e7b472f22fe99559f1 2024-11-19T08:42:47,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/645ef749cc1247a0892129e5c80b4f23 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/645ef749cc1247a0892129e5c80b4f23 2024-11-19T08:42:47,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d54753bb3b14441294b514fe427f6d96 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/d54753bb3b14441294b514fe427f6d96 2024-11-19T08:42:47,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/77d0aeeb138c4698877e1ab5340f094d to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/77d0aeeb138c4698877e1ab5340f094d 2024-11-19T08:42:47,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/dae84dc8d20d464ba48c1e8ed2fb5b70 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/dae84dc8d20d464ba48c1e8ed2fb5b70 2024-11-19T08:42:47,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/1c584e6c9a5f49049326b0b6cc022b78 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/1c584e6c9a5f49049326b0b6cc022b78 2024-11-19T08:42:47,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6bcb96c4f999487aa9221de0f14079ae to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/6bcb96c4f999487aa9221de0f14079ae 2024-11-19T08:42:47,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/29de1e1412834177a0dc1783d3169644 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/29de1e1412834177a0dc1783d3169644 2024-11-19T08:42:47,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b20799ce03f24a16bb6f2e26b91454bb to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/b20799ce03f24a16bb6f2e26b91454bb 2024-11-19T08:42:47,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/feb8c8f1314d4b14a851e12d54df56bd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/feb8c8f1314d4b14a851e12d54df56bd 2024-11-19T08:42:47,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5c2d19d5354740a18f46226539ae5f35, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/c8253e54db1d447c8e1de9b947322b95, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08efffeba9f94eb796fc33443f3e38ec, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d2573f18c6a745bc8da009b5983367db, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587298349b8742b49833baf3180a362f, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6cc49563f8934197acdf1160000b3179, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cfcfac7ca8549c7895b04d87107d703, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/82dfa7e6449d41f7b0849e5b9f92011e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d3c941158d7b4667a11730203d1d6b03, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587415305c744e26a4c31c57fc7f9fac, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08537430a25647f38cabe089a1f12ce2, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0f7b197ee4b3447ab59f1fffe84c7520, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f7703581d3a2463a9bd37657cc06c584, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/df3979c8bb574a7c81461442e0035471, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/50632de3d3e844c9bb148ecf7e2b074c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d7ed9fc9029b4ee7b11767bd939fff92, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5cf712a0565c4d84ba789d000f88fe77, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/cf42f9f3760b428d8217e777b838e54c, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6380d86c93e649438165050d797fcc4a, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f2a1ad76243549479e5ffe2da342f2dd, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0af967426a994e0e87856db34b54b29e, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cdf6c8c02ac423ca209afe064fffef6] to archive 2024-11-19T08:42:47,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T08:42:47,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5c2d19d5354740a18f46226539ae5f35 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5c2d19d5354740a18f46226539ae5f35 2024-11-19T08:42:47,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/c8253e54db1d447c8e1de9b947322b95 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/c8253e54db1d447c8e1de9b947322b95 2024-11-19T08:42:47,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08efffeba9f94eb796fc33443f3e38ec to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08efffeba9f94eb796fc33443f3e38ec 2024-11-19T08:42:47,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d2573f18c6a745bc8da009b5983367db to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d2573f18c6a745bc8da009b5983367db 2024-11-19T08:42:47,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587298349b8742b49833baf3180a362f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587298349b8742b49833baf3180a362f 2024-11-19T08:42:47,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6cc49563f8934197acdf1160000b3179 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6cc49563f8934197acdf1160000b3179 2024-11-19T08:42:47,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cfcfac7ca8549c7895b04d87107d703 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cfcfac7ca8549c7895b04d87107d703 2024-11-19T08:42:47,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/82dfa7e6449d41f7b0849e5b9f92011e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/82dfa7e6449d41f7b0849e5b9f92011e 2024-11-19T08:42:47,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d3c941158d7b4667a11730203d1d6b03 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d3c941158d7b4667a11730203d1d6b03 2024-11-19T08:42:47,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587415305c744e26a4c31c57fc7f9fac to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/587415305c744e26a4c31c57fc7f9fac 2024-11-19T08:42:47,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08537430a25647f38cabe089a1f12ce2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/08537430a25647f38cabe089a1f12ce2 2024-11-19T08:42:47,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0f7b197ee4b3447ab59f1fffe84c7520 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0f7b197ee4b3447ab59f1fffe84c7520 2024-11-19T08:42:47,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f7703581d3a2463a9bd37657cc06c584 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f7703581d3a2463a9bd37657cc06c584 2024-11-19T08:42:47,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/df3979c8bb574a7c81461442e0035471 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/df3979c8bb574a7c81461442e0035471 2024-11-19T08:42:47,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/50632de3d3e844c9bb148ecf7e2b074c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/50632de3d3e844c9bb148ecf7e2b074c 2024-11-19T08:42:47,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d7ed9fc9029b4ee7b11767bd939fff92 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/d7ed9fc9029b4ee7b11767bd939fff92 2024-11-19T08:42:47,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5cf712a0565c4d84ba789d000f88fe77 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/5cf712a0565c4d84ba789d000f88fe77 2024-11-19T08:42:47,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/cf42f9f3760b428d8217e777b838e54c to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/cf42f9f3760b428d8217e777b838e54c 2024-11-19T08:42:47,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6380d86c93e649438165050d797fcc4a to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/6380d86c93e649438165050d797fcc4a 2024-11-19T08:42:47,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f2a1ad76243549479e5ffe2da342f2dd to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/f2a1ad76243549479e5ffe2da342f2dd 2024-11-19T08:42:47,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0af967426a994e0e87856db34b54b29e to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0af967426a994e0e87856db34b54b29e 2024-11-19T08:42:47,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cdf6c8c02ac423ca209afe064fffef6 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/0cdf6c8c02ac423ca209afe064fffef6 2024-11-19T08:42:47,297 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-11-19T08:42:47,297 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. 2024-11-19T08:42:47,298 DEBUG [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for f0224c56755d95b541219e376c56cd39: Waiting for close lock at 1732005765381Running coprocessor pre-close hooks at 1732005765381Disabling compacts and flushes for region at 1732005765381Disabling writes for close at 1732005765543 (+162 ms)Obtaining lock to block concurrent updates at 1732005765543Preparing flush snapshotting stores in f0224c56755d95b541219e376c56cd39 at 1732005765543Finished memstore snapshotting TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732005765543Flushing stores of TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39. at 1732005765543Flushing f0224c56755d95b541219e376c56cd39/A: creating writer at 1732005765543Flushing mob file f0224c56755d95b541219e376c56cd39/A: appending metadata at 1732005765547 (+4 ms)Flushing mob file f0224c56755d95b541219e376c56cd39/A: closing flushed file at 1732005765547Flushing f0224c56755d95b541219e376c56cd39/A: appending metadata at 1732005765958 (+411 ms)Flushing f0224c56755d95b541219e376c56cd39/A: closing flushed file at 1732005765958Flushing f0224c56755d95b541219e376c56cd39/B: creating writer at 1732005766366 (+408 ms)Flushing f0224c56755d95b541219e376c56cd39/B: appending metadata at 1732005766368 (+2 ms)Flushing f0224c56755d95b541219e376c56cd39/B: closing flushed file at 1732005766369 (+1 ms)Flushing f0224c56755d95b541219e376c56cd39/C: creating writer at 1732005766781 (+412 ms)Flushing f0224c56755d95b541219e376c56cd39/C: appending metadata at 1732005766784 (+3 ms)Flushing f0224c56755d95b541219e376c56cd39/C: closing flushed file at 1732005766784Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@199272aa: reopening flushed file at 1732005767213 (+429 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f596416: reopening flushed file at 1732005767217 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35e3a95: reopening flushed file at 1732005767221 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f0224c56755d95b541219e376c56cd39 in 1681ms, sequenceid=322, compaction requested=false at 1732005767225 (+4 ms)Writing region close event to WAL at 1732005767295 (+70 ms)Running coprocessor post-close hooks at 1732005767297 (+2 ms)Closed at 1732005767297 2024-11-19T08:42:47,299 INFO [RS_CLOSE_REGION-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,300 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=f0224c56755d95b541219e376c56cd39, regionState=CLOSED 2024-11-19T08:42:47,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 because future has completed 2024-11-19T08:42:47,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-19T08:42:47,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure f0224c56755d95b541219e376c56cd39, server=3ab37fa97a98,34165,1732005593269 in 2.0730 sec 2024-11-19T08:42:47,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-19T08:42:47,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0224c56755d95b541219e376c56cd39, UNASSIGN in 2.0780 sec 2024-11-19T08:42:47,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-19T08:42:47,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0800 sec 2024-11-19T08:42:47,305 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732005767305"}]},"ts":"1732005767305"} 2024-11-19T08:42:47,306 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T08:42:47,306 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T08:42:47,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0880 sec 2024-11-19T08:42:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T08:42:47,367 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:47,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T08:42:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:47,371 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T08:42:47,372 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:47,375 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,378 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C, FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/recovered.edits] 2024-11-19T08:42:47,381 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/c7d4051aac3d4ff386069d89f74806a2 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/c7d4051aac3d4ff386069d89f74806a2 2024-11-19T08:42:47,382 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e7cb9a65b2e8478197712a52d90b4f69 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/A/e7cb9a65b2e8478197712a52d90b4f69 2024-11-19T08:42:47,383 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/393a7df34c75419cafc47786ec0d3b55 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/393a7df34c75419cafc47786ec0d3b55 2024-11-19T08:42:47,384 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/575151f9e3fe45ec8bbf83e045144e4f to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/B/575151f9e3fe45ec8bbf83e045144e4f 2024-11-19T08:42:47,386 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/8bb29edac3b745cb8276cf09877a3151 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/8bb29edac3b745cb8276cf09877a3151 2024-11-19T08:42:47,387 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/e0080e0c2de24648aa6c30d2c3b690d4 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/C/e0080e0c2de24648aa6c30d2c3b690d4 2024-11-19T08:42:47,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/recovered.edits/325.seqid to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39/recovered.edits/325.seqid 2024-11-19T08:42:47,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/default/TestAcidGuarantees/f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,389 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T08:42:47,389 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T08:42:47,390 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T08:42:47,392 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191e64c3019045460bb7e3637d500cb782_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191e64c3019045460bb7e3637d500cb782_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,393 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111920e2454018db4ae68495450239240af3_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111920e2454018db4ae68495450239240af3_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,394 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119326035270db94a1c89093247ae3990bd_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119326035270db94a1c89093247ae3990bd_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,395 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111937b177a3d6004f7da21fb9d98990a751_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111937b177a3d6004f7da21fb9d98990a751_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,396 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111938cfaf31f13e43e6b7c1e6d97ddc4504_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111938cfaf31f13e43e6b7c1e6d97ddc4504_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,396 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119583de29e8eca497da18796bec58fd8ce_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119583de29e8eca497da18796bec58fd8ce_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,397 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195f02a11471624dc9b7ab0c608124e6b8_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195f02a11471624dc9b7ab0c608124e6b8_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,398 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119667e13ba63a34633950f072eea2f11b0_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119667e13ba63a34633950f072eea2f11b0_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,399 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111968654042d91c4c92b2757110a1ef98c2_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111968654042d91c4c92b2757110a1ef98c2_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,400 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196efaabda90ed4604a2f357b433e09c8e_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196efaabda90ed4604a2f357b433e09c8e_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,401 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119762e9380b118459ca04e4b3575e5dbf0_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119762e9380b118459ca04e4b3575e5dbf0_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,402 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2602fd4243d41b99160aee425e36b5b_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a2602fd4243d41b99160aee425e36b5b_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,402 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afcddc58bb8b4b19b0fbe97c3e663a03_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119afcddc58bb8b4b19b0fbe97c3e663a03_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,403 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b2570b6f455746e09e727b349c40ac7e_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b2570b6f455746e09e727b349c40ac7e_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,404 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c4a02cdd86814456b14fab7de5cb68f8_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c4a02cdd86814456b14fab7de5cb68f8_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,404 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dff135b093e24b239e32fa8b83b2966d_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dff135b093e24b239e32fa8b83b2966d_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e49a1040f2004738a9af0fd649d5bc3e_f0224c56755d95b541219e376c56cd39 to hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e49a1040f2004738a9af0fd649d5bc3e_f0224c56755d95b541219e376c56cd39 2024-11-19T08:42:47,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T08:42:47,407 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:47,409 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T08:42:47,410 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T08:42:47,411 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:47,411 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T08:42:47,412 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732005767411"}]},"ts":"9223372036854775807"} 2024-11-19T08:42:47,413 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T08:42:47,413 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f0224c56755d95b541219e376c56cd39, NAME => 'TestAcidGuarantees,,1732005739433.f0224c56755d95b541219e376c56cd39.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T08:42:47,413 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T08:42:47,413 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732005767413"}]},"ts":"9223372036854775807"} 2024-11-19T08:42:47,414 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T08:42:47,415 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T08:42:47,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-19T08:42:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36325 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T08:42:47,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T08:42:47,476 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T08:42:47,484 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 238) - Thread LEAK? -, OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=5450 (was 5786) 2024-11-19T08:42:47,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T08:42:47,485 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:42:47,485 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:42:47,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:47,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:47,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T08:42:47,485 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T08:42:47,485 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=350595032, stopped=false 2024-11-19T08:42:47,486 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3ab37fa97a98,36325,1732005592414 2024-11-19T08:42:47,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T08:42:47,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T08:42:47,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:42:47,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:42:47,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T08:42:47,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T08:42:47,505 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T08:42:47,505 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:42:47,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T08:42:47,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:47,505 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3ab37fa97a98,34165,1732005593269' ***** 2024-11-19T08:42:47,506 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T08:42:47,506 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T08:42:47,506 INFO [RS:0;3ab37fa97a98:34165 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T08:42:47,506 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T08:42:47,506 INFO [RS:0;3ab37fa97a98:34165 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T08:42:47,506 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(959): stopping server 3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:47,506 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T08:42:47,506 INFO [RS:0;3ab37fa97a98:34165 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3ab37fa97a98:34165. 2024-11-19T08:42:47,506 DEBUG [RS:0;3ab37fa97a98:34165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T08:42:47,506 DEBUG [RS:0;3ab37fa97a98:34165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:47,507 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T08:42:47,507 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T08:42:47,507 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T08:42:47,507 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T08:42:47,507 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T08:42:47,507 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T08:42:47,507 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:47,507 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T08:42:47,507 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T08:42:47,507 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T08:42:47,508 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T08:42:47,508 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T08:42:47,508 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T08:42:47,525 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/info/eab4818732a24ace83def5a45c1e215e is 95, key is TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da./info:/1732005624170/DeleteFamily/seqid=0 2024-11-19T08:42:47,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742419_1595 (size=6514) 2024-11-19T08:42:47,533 INFO [regionserver/3ab37fa97a98:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T08:42:47,708 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:47,908 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:47,930 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/info/eab4818732a24ace83def5a45c1e215e 2024-11-19T08:42:47,956 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/ns/53cef864bd434029bd8c76010ff11480 is 93, key is TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da./ns:/1732005624115/DeleteFamily/seqid=0 2024-11-19T08:42:47,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742420_1596 (size=5906) 2024-11-19T08:42:48,108 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:48,309 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:48,360 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/ns/53cef864bd434029bd8c76010ff11480 2024-11-19T08:42:48,385 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/rep_barrier/02338967c38245899210956b245ae9ce is 102, key is TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da./rep_barrier:/1732005624115/DeleteFamily/seqid=0 2024-11-19T08:42:48,386 INFO [regionserver/3ab37fa97a98:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T08:42:48,386 INFO [regionserver/3ab37fa97a98:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T08:42:48,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742421_1597 (size=6025) 2024-11-19T08:42:48,509 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:48,709 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:48,790 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/rep_barrier/02338967c38245899210956b245ae9ce 2024-11-19T08:42:48,818 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/table/49cdde3d3c4942be8ab7e3f308422490 is 96, key is TestAcidGuarantees,,1732005598028.ca9bd3485db2e8c855878994330f02da./table:/1732005624115/DeleteFamily/seqid=0 2024-11-19T08:42:48,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742422_1598 (size=5936) 2024-11-19T08:42:48,909 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:49,110 DEBUG [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T08:42:49,223 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/table/49cdde3d3c4942be8ab7e3f308422490 2024-11-19T08:42:49,233 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/info/eab4818732a24ace83def5a45c1e215e as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/info/eab4818732a24ace83def5a45c1e215e 2024-11-19T08:42:49,237 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/info/eab4818732a24ace83def5a45c1e215e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T08:42:49,237 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/ns/53cef864bd434029bd8c76010ff11480 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/ns/53cef864bd434029bd8c76010ff11480 2024-11-19T08:42:49,241 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/ns/53cef864bd434029bd8c76010ff11480, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T08:42:49,241 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/rep_barrier/02338967c38245899210956b245ae9ce as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/rep_barrier/02338967c38245899210956b245ae9ce 2024-11-19T08:42:49,244 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/rep_barrier/02338967c38245899210956b245ae9ce, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T08:42:49,245 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/.tmp/table/49cdde3d3c4942be8ab7e3f308422490 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/table/49cdde3d3c4942be8ab7e3f308422490 2024-11-19T08:42:49,247 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/table/49cdde3d3c4942be8ab7e3f308422490, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T08:42:49,248 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-19T08:42:49,251 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T08:42:49,252 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T08:42:49,252 INFO [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T08:42:49,252 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732005767507Running coprocessor pre-close hooks at 1732005767507Disabling compacts and flushes for region at 1732005767507Disabling writes for close at 1732005767508 (+1 ms)Obtaining lock to block concurrent updates at 1732005767508Preparing flush snapshotting stores in 1588230740 at 1732005767508Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732005767508Flushing stores of hbase:meta,,1.1588230740 at 1732005767508Flushing 1588230740/info: creating writer at 1732005767508Flushing 1588230740/info: appending metadata at 1732005767525 (+17 ms)Flushing 1588230740/info: closing flushed file at 1732005767525Flushing 1588230740/ns: creating writer at 1732005767940 (+415 ms)Flushing 1588230740/ns: appending metadata at 1732005767956 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732005767956Flushing 1588230740/rep_barrier: creating writer at 1732005768366 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732005768385 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732005768385Flushing 1588230740/table: creating writer at 1732005768800 (+415 ms)Flushing 1588230740/table: appending metadata at 1732005768818 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732005768818Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4249139d: reopening flushed file at 1732005769232 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15eeaea8: reopening flushed file at 1732005769237 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45426251: reopening flushed file at 1732005769241 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cf1fdf8: reopening flushed file at 1732005769244 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1732005769248 (+4 ms)Writing region close event to WAL at 1732005769249 (+1 ms)Running coprocessor post-close hooks at 1732005769252 (+3 ms)Closed at 1732005769252 2024-11-19T08:42:49,252 DEBUG [RS_CLOSE_META-regionserver/3ab37fa97a98:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T08:42:49,310 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(976): stopping server 3ab37fa97a98,34165,1732005593269; all regions closed. 2024-11-19T08:42:49,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741834_1010 (size=25344) 2024-11-19T08:42:49,319 DEBUG [RS:0;3ab37fa97a98:34165 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/oldWALs 2024-11-19T08:42:49,320 INFO [RS:0;3ab37fa97a98:34165 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3ab37fa97a98%2C34165%2C1732005593269.meta:.meta(num 1732005597505) 2024-11-19T08:42:49,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741832_1008 (size=13769712) 2024-11-19T08:42:49,323 DEBUG [RS:0;3ab37fa97a98:34165 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/oldWALs 2024-11-19T08:42:49,323 INFO [RS:0;3ab37fa97a98:34165 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3ab37fa97a98%2C34165%2C1732005593269:(num 1732005596544) 2024-11-19T08:42:49,323 DEBUG [RS:0;3ab37fa97a98:34165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T08:42:49,323 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T08:42:49,323 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T08:42:49,324 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.ChoreService(370): Chore service for: regionserver/3ab37fa97a98:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T08:42:49,324 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T08:42:49,324 INFO [regionserver/3ab37fa97a98:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T08:42:49,324 INFO [RS:0;3ab37fa97a98:34165 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34165 2024-11-19T08:42:49,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3ab37fa97a98,34165,1732005593269 2024-11-19T08:42:49,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T08:42:49,384 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T08:42:49,395 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3ab37fa97a98,34165,1732005593269] 2024-11-19T08:42:49,404 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3ab37fa97a98,34165,1732005593269 already deleted, retry=false 2024-11-19T08:42:49,404 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3ab37fa97a98,34165,1732005593269 expired; onlineServers=0 2024-11-19T08:42:49,405 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3ab37fa97a98,36325,1732005592414' ***** 2024-11-19T08:42:49,405 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T08:42:49,405 INFO [M:0;3ab37fa97a98:36325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T08:42:49,405 INFO [M:0;3ab37fa97a98:36325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T08:42:49,405 DEBUG [M:0;3ab37fa97a98:36325 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T08:42:49,405 DEBUG [M:0;3ab37fa97a98:36325 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T08:42:49,405 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T08:42:49,405 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster-HFileCleaner.small.0-1732005596209 {}] cleaner.HFileCleaner(306): Exit Thread[master/3ab37fa97a98:0:becomeActiveMaster-HFileCleaner.small.0-1732005596209,5,FailOnTimeoutGroup] 2024-11-19T08:42:49,405 DEBUG [master/3ab37fa97a98:0:becomeActiveMaster-HFileCleaner.large.0-1732005596208 {}] cleaner.HFileCleaner(306): Exit Thread[master/3ab37fa97a98:0:becomeActiveMaster-HFileCleaner.large.0-1732005596208,5,FailOnTimeoutGroup] 2024-11-19T08:42:49,406 INFO [M:0;3ab37fa97a98:36325 {}] hbase.ChoreService(370): Chore service for: master/3ab37fa97a98:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T08:42:49,406 INFO [M:0;3ab37fa97a98:36325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T08:42:49,406 DEBUG [M:0;3ab37fa97a98:36325 {}] master.HMaster(1795): Stopping service threads 2024-11-19T08:42:49,406 INFO [M:0;3ab37fa97a98:36325 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T08:42:49,406 INFO [M:0;3ab37fa97a98:36325 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T08:42:49,407 ERROR [M:0;3ab37fa97a98:36325 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:44959 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44959,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T08:42:49,408 INFO [M:0;3ab37fa97a98:36325 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T08:42:49,408 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T08:42:49,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T08:42:49,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T08:42:49,418 DEBUG [M:0;3ab37fa97a98:36325 {}] zookeeper.ZKUtil(347): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T08:42:49,418 WARN [M:0;3ab37fa97a98:36325 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T08:42:49,419 INFO [M:0;3ab37fa97a98:36325 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/.lastflushedseqids 2024-11-19T08:42:49,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742423_1599 (size=72) 2024-11-19T08:42:49,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T08:42:49,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34165-0x10153898cdd0001, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T08:42:49,497 INFO [RS:0;3ab37fa97a98:34165 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T08:42:49,497 INFO [RS:0;3ab37fa97a98:34165 {}] regionserver.HRegionServer(1031): Exiting; stopping=3ab37fa97a98,34165,1732005593269; zookeeper connection closed. 2024-11-19T08:42:49,497 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@281a2a86 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@281a2a86 2024-11-19T08:42:49,498 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T08:42:49,835 INFO [M:0;3ab37fa97a98:36325 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T08:42:49,835 INFO [M:0;3ab37fa97a98:36325 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T08:42:49,836 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T08:42:49,836 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:42:49,836 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:42:49,836 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T08:42:49,836 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:42:49,836 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=771.51 KB heapSize=949.02 KB 2024-11-19T08:42:49,857 DEBUG [M:0;3ab37fa97a98:36325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f278d1b7c8bb450caf3b8355df8dec81 is 82, key is hbase:meta,,1/info:regioninfo/1732005597592/Put/seqid=0 2024-11-19T08:42:49,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742424_1600 (size=5672) 2024-11-19T08:42:50,262 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2193 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f278d1b7c8bb450caf3b8355df8dec81 2024-11-19T08:42:50,298 DEBUG [M:0;3ab37fa97a98:36325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00083792381044519ff9d387b88c4a07 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x99/proc:d/1732005742518/Put/seqid=0 2024-11-19T08:42:50,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742425_1601 (size=45172) 2024-11-19T08:42:50,705 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770.96 KB at sequenceid=2193 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00083792381044519ff9d387b88c4a07 2024-11-19T08:42:50,714 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00083792381044519ff9d387b88c4a07 2024-11-19T08:42:50,731 DEBUG [M:0;3ab37fa97a98:36325 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/91430e53de0f45a5beec8c4521d4a33c is 69, key is 3ab37fa97a98,34165,1732005593269/rs:state/1732005596285/Put/seqid=0 2024-11-19T08:42:50,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073742426_1602 (size=5156) 2024-11-19T08:42:50,734 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2193 (bloomFilter=true), to=hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/91430e53de0f45a5beec8c4521d4a33c 2024-11-19T08:42:50,737 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f278d1b7c8bb450caf3b8355df8dec81 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f278d1b7c8bb450caf3b8355df8dec81 2024-11-19T08:42:50,741 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f278d1b7c8bb450caf3b8355df8dec81, entries=8, sequenceid=2193, filesize=5.5 K 2024-11-19T08:42:50,741 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00083792381044519ff9d387b88c4a07 as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00083792381044519ff9d387b88c4a07 2024-11-19T08:42:50,744 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00083792381044519ff9d387b88c4a07 2024-11-19T08:42:50,744 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00083792381044519ff9d387b88c4a07, entries=176, sequenceid=2193, filesize=44.1 K 2024-11-19T08:42:50,744 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/91430e53de0f45a5beec8c4521d4a33c as hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/91430e53de0f45a5beec8c4521d4a33c 2024-11-19T08:42:50,747 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44959/user/jenkins/test-data/1ad1cd89-f39e-115f-09a8-c3c42da0766a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/91430e53de0f45a5beec8c4521d4a33c, entries=1, sequenceid=2193, filesize=5.0 K 2024-11-19T08:42:50,748 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(3140): Finished flush of dataSize ~771.51 KB/790027, heapSize ~948.72 KB/971488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 912ms, sequenceid=2193, compaction requested=false 2024-11-19T08:42:50,749 INFO [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T08:42:50,749 DEBUG [M:0;3ab37fa97a98:36325 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732005769835Disabling compacts and flushes for region at 1732005769835Disabling writes for close at 1732005769836 (+1 ms)Obtaining lock to block concurrent updates at 1732005769836Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732005769836Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=790027, getHeapSize=971728, getOffHeapSize=0, getCellsCount=2394 at 1732005769837 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732005769839 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732005769839Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732005769857 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732005769857Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732005770276 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732005770298 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732005770298Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732005770714 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732005770730 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732005770730Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@360a596d: reopening flushed file at 1732005770737 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6060dc92: reopening flushed file at 1732005770741 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ca5490a: reopening flushed file at 1732005770744 (+3 ms)Finished flush of dataSize ~771.51 KB/790027, heapSize ~948.72 KB/971488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 912ms, sequenceid=2193, compaction requested=false at 1732005770748 (+4 ms)Writing region close event to WAL at 1732005770749 (+1 ms)Closed at 1732005770749 2024-11-19T08:42:50,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36115 is added to blk_1073741830_1006 (size=933795) 2024-11-19T08:42:50,752 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T08:42:50,752 INFO [M:0;3ab37fa97a98:36325 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T08:42:50,752 INFO [M:0;3ab37fa97a98:36325 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36325 2024-11-19T08:42:50,752 INFO [M:0;3ab37fa97a98:36325 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T08:42:50,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T08:42:50,888 INFO [M:0;3ab37fa97a98:36325 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T08:42:50,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36325-0x10153898cdd0000, quorum=127.0.0.1:55285, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T08:42:50,893 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T08:42:50,895 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T08:42:50,895 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T08:42:50,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T08:42:50,895 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/hadoop.log.dir/,STOPPED} 2024-11-19T08:42:50,898 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T08:42:50,898 WARN [BP-951815767-172.17.0.2-1732005588737 heartbeating to localhost/127.0.0.1:44959 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T08:42:50,898 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T08:42:50,898 WARN [BP-951815767-172.17.0.2-1732005588737 heartbeating to localhost/127.0.0.1:44959 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-951815767-172.17.0.2-1732005588737 (Datanode Uuid 41be1e1c-e9a1-44c9-893b-730bc6b67745) service to localhost/127.0.0.1:44959 2024-11-19T08:42:50,900 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/data/data1/current/BP-951815767-172.17.0.2-1732005588737 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T08:42:50,900 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/cluster_49a3fc04-d057-7c9a-6140-ccedd2e88f07/data/data2/current/BP-951815767-172.17.0.2-1732005588737 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T08:42:50,901 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T08:42:50,907 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T08:42:50,907 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T08:42:50,907 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T08:42:50,907 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T08:42:50,907 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/732dfa93-2151-17bf-17e3-99dbeb8165ea/hadoop.log.dir/,STOPPED} 2024-11-19T08:42:50,920 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T08:42:51,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down